2024-11-19 12:54:36,221 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 12:54:36,239 main DEBUG Took 0.016563 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 12:54:36,239 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 12:54:36,240 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 12:54:36,241 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 12:54:36,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,248 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 12:54:36,260 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,263 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,264 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,265 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,266 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,266 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,267 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,268 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,268 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,269 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,269 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,270 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,271 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 12:54:36,271 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,272 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 12:54:36,273 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 12:54:36,274 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 12:54:36,276 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 12:54:36,276 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 12:54:36,277 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 12:54:36,278 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 12:54:36,285 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 12:54:36,288 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 12:54:36,290 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 12:54:36,291 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 12:54:36,291 main DEBUG createAppenders(={Console}) 2024-11-19 12:54:36,292 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-19 12:54:36,293 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 12:54:36,293 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-19 12:54:36,294 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 12:54:36,294 main DEBUG OutputStream closed 2024-11-19 12:54:36,295 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 12:54:36,295 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 12:54:36,295 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-19 12:54:36,365 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 12:54:36,367 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 12:54:36,368 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 12:54:36,369 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 12:54:36,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 12:54:36,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 12:54:36,370 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 12:54:36,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 12:54:36,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 12:54:36,371 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 12:54:36,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 12:54:36,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 12:54:36,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 12:54:36,372 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 12:54:36,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 12:54:36,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 12:54:36,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 12:54:36,374 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 12:54:36,376 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 12:54:36,376 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-19 12:54:36,377 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 12:54:36,377 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-19T12:54:36,390 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-19 12:54:36,393 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 12:54:36,393 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T12:54:36,611 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5 2024-11-19T12:54:36,634 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=502, ProcessCount=11, AvailableMemoryMB=7235 2024-11-19T12:54:36,723 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T12:54:36,804 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:36,817 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/cluster_dcaea1de-3159-f659-37b7-90024f8a3514, deleteOnExit=true 2024-11-19T12:54:36,944 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/cluster_dcaea1de-3159-f659-37b7-90024f8a3514/zookeeper_0, clientPort=65054, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/cluster_dcaea1de-3159-f659-37b7-90024f8a3514/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/cluster_dcaea1de-3159-f659-37b7-90024f8a3514/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T12:54:36,955 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65054 2024-11-19T12:54:36,955 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:37,036 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 with version=8 2024-11-19T12:54:37,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase-staging 2024-11-19T12:54:37,128 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T12:54:37,364 INFO [Time-limited test {}] client.ConnectionUtils(128): master/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:37,376 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:37,378 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:37,389 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:37,389 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:37,390 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:37,522 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T12:54:37,612 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:54:37,646 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T12:54:37,656 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T12:54:37,659 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:37,698 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96048 (auto-detected) 2024-11-19T12:54:37,699 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T12:54:37,723 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36083 2024-11-19T12:54:37,746 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36083 connecting to ZooKeeper ensemble=127.0.0.1:65054 2024-11-19T12:54:37,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360830x0, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:37,871 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36083-0x1015472c93e0000 connected 2024-11-19T12:54:37,972 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:37,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:37,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:37,987 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697, hbase.cluster.distributed=false 2024-11-19T12:54:38,009 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:38,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-19T12:54:38,014 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36083 2024-11-19T12:54:38,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36083 2024-11-19T12:54:38,015 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-19T12:54:38,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-19T12:54:38,091 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:38,092 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,093 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:38,096 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:54:38,099 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:38,100 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37283 2024-11-19T12:54:38,102 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37283 connecting to ZooKeeper ensemble=127.0.0.1:65054 2024-11-19T12:54:38,102 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:38,103 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:38,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372830x0, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:38,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:372830x0, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:38,128 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37283-0x1015472c93e0001 connected 2024-11-19T12:54:38,135 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:54:38,147 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.cache 2024-11-19T12:54:38,153 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T12:54:38,154 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T12:54:38,155 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T12:54:38,157 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T12:54:38,157 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T12:54:38,164 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T12:54:38,168 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T12:54:38,169 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence is complete. 2024-11-19T12:54:38,170 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:54:38,173 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:54:38,179 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:38,180 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37283 2024-11-19T12:54:38,182 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37283 2024-11-19T12:54:38,183 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37283 2024-11-19T12:54:38,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37283 2024-11-19T12:54:38,189 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37283 2024-11-19T12:54:38,209 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:38,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,209 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:38,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:38,210 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:38,210 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:54:38,211 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:38,211 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44249 2024-11-19T12:54:38,213 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44249 connecting to ZooKeeper ensemble=127.0.0.1:65054 2024-11-19T12:54:38,214 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:38,215 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:38,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442490x0, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:38,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442490x0, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:38,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44249-0x1015472c93e0002 connected 2024-11-19T12:54:38,245 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:54:38,246 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.cache 2024-11-19T12:54:38,247 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T12:54:38,247 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T12:54:38,247 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T12:54:38,248 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T12:54:38,248 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T12:54:38,248 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T12:54:38,249 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:54:38,249 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T12:54:38,249 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence is complete. 2024-11-19T12:54:38,250 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:54:38,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:38,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-19T12:54:38,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44249 2024-11-19T12:54:38,256 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44249 2024-11-19T12:54:38,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-19T12:54:38,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44249 2024-11-19T12:54:38,279 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aba5a916dfea:36083 2024-11-19T12:54:38,280 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aba5a916dfea,36083,1732020877205 2024-11-19T12:54:38,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,296 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aba5a916dfea,36083,1732020877205 2024-11-19T12:54:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,327 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,328 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T12:54:38,330 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aba5a916dfea,36083,1732020877205 from backup master directory 2024-11-19T12:54:38,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aba5a916dfea,36083,1732020877205 2024-11-19T12:54:38,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:38,341 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:38,341 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aba5a916dfea,36083,1732020877205 2024-11-19T12:54:38,344 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T12:54:38,347 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T12:54:38,356 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase.id] with ID: caae1387-24b3-4c48-9ef1-c5908b4345cc 2024-11-19T12:54:38,357 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.tmp/hbase.id 2024-11-19T12:54:38,362 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase.id] 2024-11-19T12:54:38,402 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:38,408 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T12:54:38,427 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-19T12:54:38,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:38,470 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:38,473 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T12:54:38,513 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:38,518 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:38,574 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store 2024-11-19T12:54:38,597 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T12:54:38,600 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:38,602 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:54:38,603 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:38,603 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:38,605 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:54:38,605 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:38,606 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:38,607 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020878602Disabling compacts and flushes for region at 1732020878602Disabling writes for close at 1732020878605 (+3 ms)Writing region close event to WAL at 1732020878606 (+1 ms)Closed at 1732020878606 2024-11-19T12:54:38,607 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/.initializing 2024-11-19T12:54:38,608 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36083,1732020877205 2024-11-19T12:54:38,620 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:38,635 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aba5a916dfea%2C36083%2C1732020877205, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36083,1732020877205, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/oldWALs, maxLogs=10 2024-11-19T12:54:38,678 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36083,1732020877205/aba5a916dfea%2C36083%2C1732020877205.1732020878641 2024-11-19T12:54:38,679 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:38,679 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:38,680 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:38,684 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,685 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,721 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T12:54:38,754 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:38,756 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:38,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T12:54:38,762 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:38,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:38,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T12:54:38,768 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:38,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:38,769 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T12:54:38,774 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:38,775 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:38,775 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,779 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,779 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,782 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,783 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,786 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:38,788 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:38,795 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:38,796 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69602008, jitterRate=0.03715074062347412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:38,803 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732020878698Initializing all the Stores at 1732020878698Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020878699 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020878699Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020878700 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020878700Cleaning up temporary data from old regions at 1732020878783 (+83 ms)Region opened successfully at 1732020878803 (+20 ms) 2024-11-19T12:54:38,804 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T12:54:38,846 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9b52c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:38,889 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T12:54:38,905 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T12:54:38,905 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T12:54:38,910 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:54:38,911 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-19T12:54:38,918 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-19T12:54:38,919 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T12:54:38,952 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T12:54:38,965 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T12:54:39,006 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T12:54:39,009 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T12:54:39,011 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T12:54:39,018 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T12:54:39,020 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T12:54:39,026 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T12:54:39,034 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T12:54:39,036 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T12:54:39,043 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T12:54:39,060 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T12:54:39,068 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,095 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aba5a916dfea,36083,1732020877205, sessionid=0x1015472c93e0000, setting cluster-up flag (Was=false) 2024-11-19T12:54:39,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,151 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T12:54:39,154 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36083,1732020877205 2024-11-19T12:54:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,201 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T12:54:39,204 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36083,1732020877205 2024-11-19T12:54:39,210 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T12:54:39,267 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(746): ClusterId : caae1387-24b3-4c48-9ef1-c5908b4345cc 2024-11-19T12:54:39,267 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(746): ClusterId : caae1387-24b3-4c48-9ef1-c5908b4345cc 2024-11-19T12:54:39,270 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:54:39,270 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:54:39,293 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:39,303 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T12:54:39,311 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T12:54:39,311 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:54:39,311 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:54:39,312 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:54:39,312 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:54:39,317 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aba5a916dfea,36083,1732020877205 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T12:54:39,333 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:54:39,333 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:54:39,333 DEBUG [RS:1;aba5a916dfea:44249 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72bb514c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:39,333 DEBUG [RS:0;aba5a916dfea:37283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76bba215, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:39,334 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:39,335 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:39,335 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:39,335 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:39,335 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aba5a916dfea:0, corePoolSize=10, maxPoolSize=10 2024-11-19T12:54:39,336 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,336 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:39,336 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,342 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:39,342 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T12:54:39,344 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732020909344 2024-11-19T12:54:39,346 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T12:54:39,347 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:39,347 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T12:54:39,347 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T12:54:39,349 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;aba5a916dfea:44249 2024-11-19T12:54:39,351 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T12:54:39,351 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T12:54:39,351 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T12:54:39,352 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T12:54:39,352 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aba5a916dfea:37283 2024-11-19T12:54:39,354 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:54:39,354 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:54:39,354 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:54:39,354 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:54:39,354 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:54:39,354 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:54:39,355 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,357 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36083,1732020877205 with port=44249, startcode=1732020878208 2024-11-19T12:54:39,357 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36083,1732020877205 with port=37283, startcode=1732020878058 2024-11-19T12:54:39,359 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T12:54:39,360 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T12:54:39,361 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T12:54:39,368 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T12:54:39,368 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T12:54:39,370 DEBUG [RS:0;aba5a916dfea:37283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:54:39,370 DEBUG [RS:1;aba5a916dfea:44249 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:54:39,371 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020879370,5,FailOnTimeoutGroup] 2024-11-19T12:54:39,375 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020879371,5,FailOnTimeoutGroup] 2024-11-19T12:54:39,375 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,375 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T12:54:39,377 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,377 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,379 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T12:54:39,379 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:39,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:39,395 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T12:54:39,399 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:54:39,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:39,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:39,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T12:54:39,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:54:39,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:39,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:39,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T12:54:39,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:54:39,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:39,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:39,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T12:54:39,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:54:39,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:39,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:39,414 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:54:39,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:39,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:39,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:54:39,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:54:39,421 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:39,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:54:39,430 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:39,430 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60902005, jitterRate=-0.09248940646648407}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732020879390Initializing all the Stores at 1732020879391 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020879391Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020879394 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020879394Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020879395 (+1 ms)Cleaning up temporary data from old regions at 1732020879421 (+26 ms)Region opened successfully at 1732020879435 (+14 ms) 2024-11-19T12:54:39,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:54:39,436 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:54:39,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:54:39,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:54:39,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:54:39,438 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:39,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020879435Disabling compacts and flushes for region at 1732020879435Disabling writes for close at 1732020879436 (+1 ms)Writing region close event to WAL at 1732020879437 (+1 ms)Closed at 1732020879437 2024-11-19T12:54:39,441 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:39,442 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T12:54:39,444 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40733, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:54:39,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46759, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:54:39,450 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36083 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T12:54:39,453 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36083 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,460 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:54:39,462 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36083 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,462 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T12:54:39,462 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36083 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,466 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:39,466 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:39,466 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T12:54:39,466 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T12:54:39,466 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:54:39,466 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:54:39,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:39,525 DEBUG [RS:0;aba5a916dfea:37283 {}] zookeeper.ZKUtil(111): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,525 DEBUG [RS:1;aba5a916dfea:44249 {}] zookeeper.ZKUtil(111): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,526 WARN [RS:1;aba5a916dfea:44249 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:39,526 WARN [RS:0;aba5a916dfea:37283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:39,526 INFO [RS:0;aba5a916dfea:37283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:39,526 INFO [RS:1;aba5a916dfea:44249 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:39,526 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,526 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,528 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,37283,1732020878058] 2024-11-19T12:54:39,529 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,44249,1732020878208] 2024-11-19T12:54:39,551 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:54:39,551 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:54:39,570 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:54:39,570 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:54:39,575 INFO [RS:1;aba5a916dfea:44249 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:54:39,575 INFO [RS:0;aba5a916dfea:37283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:54:39,575 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,575 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,576 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:54:39,577 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:54:39,582 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:54:39,582 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:54:39,584 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,584 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,584 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,584 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,585 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:39,585 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:39,586 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,586 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,587 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,587 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:39,587 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:39,587 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:39,587 DEBUG [RS:0;aba5a916dfea:37283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:39,587 DEBUG [RS:1;aba5a916dfea:44249 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:39,589 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,589 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,589 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,589 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,37283,1732020878058-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,590 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,44249,1732020878208-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:39,611 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:54:39,611 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:54:39,613 WARN [aba5a916dfea:36083 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T12:54:39,613 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,37283,1732020878058-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,613 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,44249,1732020878208-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,614 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,614 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,614 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.Replication(171): aba5a916dfea,44249,1732020878208 started 2024-11-19T12:54:39,614 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.Replication(171): aba5a916dfea,37283,1732020878058 started 2024-11-19T12:54:39,633 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,634 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,44249,1732020878208, RpcServer on aba5a916dfea/172.17.0.2:44249, sessionid=0x1015472c93e0002 2024-11-19T12:54:39,635 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:54:39,635 DEBUG [RS:1;aba5a916dfea:44249 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,635 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,44249,1732020878208' 2024-11-19T12:54:39,635 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:54:39,636 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:54:39,637 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:54:39,637 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:54:39,637 DEBUG [RS:1;aba5a916dfea:44249 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,44249,1732020878208 2024-11-19T12:54:39,637 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,44249,1732020878208' 2024-11-19T12:54:39,637 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:54:39,638 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:54:39,639 DEBUG [RS:1;aba5a916dfea:44249 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:54:39,639 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:39,639 INFO [RS:1;aba5a916dfea:44249 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:54:39,639 INFO [RS:1;aba5a916dfea:44249 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:54:39,639 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,37283,1732020878058, RpcServer on aba5a916dfea/172.17.0.2:37283, sessionid=0x1015472c93e0001 2024-11-19T12:54:39,639 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:54:39,640 DEBUG [RS:0;aba5a916dfea:37283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,640 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,37283,1732020878058' 2024-11-19T12:54:39,640 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:54:39,640 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:54:39,641 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:54:39,641 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:54:39,641 DEBUG [RS:0;aba5a916dfea:37283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,641 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,37283,1732020878058' 2024-11-19T12:54:39,641 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:54:39,642 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:54:39,643 DEBUG [RS:0;aba5a916dfea:37283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:54:39,643 INFO [RS:0;aba5a916dfea:37283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:54:39,643 INFO [RS:0;aba5a916dfea:37283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:54:39,745 INFO [RS:1;aba5a916dfea:44249 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:39,745 INFO [RS:0;aba5a916dfea:37283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:39,746 INFO [RS:1;aba5a916dfea:44249 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C44249%2C1732020878208, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44249,1732020878208, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:39,746 INFO [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C37283%2C1732020878058, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37283,1732020878058, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:39,767 INFO [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37283,1732020878058/aba5a916dfea%2C37283%2C1732020878058.1732020879749 2024-11-19T12:54:39,767 DEBUG [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:39,771 INFO [RS:1;aba5a916dfea:44249 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44249,1732020878208/aba5a916dfea%2C44249%2C1732020878208.1732020879749 2024-11-19T12:54:39,771 DEBUG [RS:1;aba5a916dfea:44249 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:39,869 DEBUG [aba5a916dfea:36083 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T12:54:39,883 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(204): Hosts are {aba5a916dfea=0} racks are {/default-rack=0} 2024-11-19T12:54:39,887 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:54:39,887 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:54:39,887 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:54:39,887 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:54:39,887 INFO [aba5a916dfea:36083 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:54:39,887 INFO [aba5a916dfea:36083 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:54:39,887 DEBUG [aba5a916dfea:36083 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:54:39,894 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:39,900 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,37283,1732020878058, state=OPENING 2024-11-19T12:54:39,934 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T12:54:39,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:39,944 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:39,944 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:39,944 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:39,945 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:54:39,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,37283,1732020878058}] 2024-11-19T12:54:40,120 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:54:40,123 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36267, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:54:40,130 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T12:54:40,130 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:40,132 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T12:54:40,133 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C37283%2C1732020878058.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37283,1732020878058, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:40,152 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37283,1732020878058/aba5a916dfea%2C37283%2C1732020878058.meta.1732020880135.meta 2024-11-19T12:54:40,154 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:40,154 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:40,156 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T12:54:40,158 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T12:54:40,162 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T12:54:40,166 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T12:54:40,166 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:40,166 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T12:54:40,167 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T12:54:40,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T12:54:40,169 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:54:40,169 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:40,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T12:54:40,170 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:54:40,170 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:40,171 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T12:54:40,171 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:54:40,172 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:40,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T12:54:40,173 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:54:40,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:40,173 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:54:40,174 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:40,175 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:40,175 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:54:40,175 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:54:40,175 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:40,176 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:54:40,177 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71103927, jitterRate=0.0595310777425766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:40,177 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T12:54:40,179 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732020880167Writing region info on filesystem at 1732020880167Initializing all the Stores at 1732020880168 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020880168Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020880168Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020880168Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020880168Cleaning up temporary data from old regions at 1732020880175 (+7 ms)Running coprocessor post-open hooks at 1732020880177 (+2 ms)Region opened successfully at 1732020880179 (+2 ms) 2024-11-19T12:54:40,184 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732020880110 2024-11-19T12:54:40,193 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T12:54:40,193 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T12:54:40,194 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:40,196 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,37283,1732020878058, state=OPEN 2024-11-19T12:54:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:40,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:40,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:40,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:40,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:40,227 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:40,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T12:54:40,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,37283,1732020878058 in 280 msec 2024-11-19T12:54:40,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T12:54:40,238 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 783 msec 2024-11-19T12:54:40,239 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:40,239 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T12:54:40,256 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:40,258 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,37283,1732020878058, seqNum=-1] 2024-11-19T12:54:40,275 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:40,277 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:40,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0490 sec 2024-11-19T12:54:40,297 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732020880297, completionTime=-1 2024-11-19T12:54:40,300 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T12:54:40,300 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T12:54:40,324 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T12:54:40,324 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732020940324 2024-11-19T12:54:40,324 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732021000324 2024-11-19T12:54:40,324 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-19T12:54:40,326 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T12:54:40,332 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,332 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,332 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,334 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aba5a916dfea:36083, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,334 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,335 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,341 DEBUG [master/aba5a916dfea:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T12:54:40,361 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.019sec 2024-11-19T12:54:40,362 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T12:54:40,364 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T12:54:40,365 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T12:54:40,366 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T12:54:40,366 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T12:54:40,367 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:40,368 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T12:54:40,369 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T12:54:40,370 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T12:54:40,370 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36083,1732020877205-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:40,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:40,378 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T12:54:40,378 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T12:54:40,381 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36083,-1 for getting cluster id 2024-11-19T12:54:40,384 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:40,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'caae1387-24b3-4c48-9ef1-c5908b4345cc' 2024-11-19T12:54:40,393 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:40,394 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "caae1387-24b3-4c48-9ef1-c5908b4345cc" 2024-11-19T12:54:40,396 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49244511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:40,396 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36083,-1] 2024-11-19T12:54:40,399 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:40,401 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:40,403 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:40,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:40,406 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:40,413 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,37283,1732020878058, seqNum=-1] 2024-11-19T12:54:40,414 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:40,416 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:40,447 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aba5a916dfea,36083,1732020877205 2024-11-19T12:54:40,451 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:54:40,459 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36083,1732020877205 2024-11-19T12:54:40,462 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12b12a2a 2024-11-19T12:54:40,464 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:54:40,466 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:54:40,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732020880447', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:40,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 2024-11-19T12:54:40,480 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:54:40,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732020880447" procId is: 4 2024-11-19T12:54:40,482 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,483 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:54:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:40,495 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cfd198455e201428133912868c9b07e1, NAME => 'testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732020880447', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1722): Closing cfd198455e201428133912868c9b07e1, disabling compactions & flushes 2024-11-19T12:54:40,501 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. after waiting 0 ms 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,501 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,501 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732020880447-pool-0 {}] regionserver.HRegion(1676): Region close journal for cfd198455e201428133912868c9b07e1: Waiting for close lock at 1732020880501Disabling compacts and flushes for region at 1732020880501Disabling writes for close at 1732020880501Writing region close event to WAL at 1732020880501Closed at 1732020880501 2024-11-19T12:54:40,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:54:40,507 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732020880503"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020880503"}]},"ts":"1732020880503"} 2024-11-19T12:54:40,511 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:54:40,512 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:54:40,515 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732020880447","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020880513"}]},"ts":"1732020880513"} 2024-11-19T12:54:40,523 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732020880447, state=ENABLING in hbase:meta 2024-11-19T12:54:40,524 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {aba5a916dfea=0} racks are {/default-rack=0} 2024-11-19T12:54:40,525 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:54:40,525 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:54:40,525 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:54:40,525 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:54:40,525 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:54:40,525 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:54:40,525 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:54:40,527 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, ASSIGN}] 2024-11-19T12:54:40,529 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, ASSIGN 2024-11-19T12:54:40,530 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, ASSIGN; state=OFFLINE, location=aba5a916dfea,37283,1732020878058; forceNewPlan=false, retain=false 2024-11-19T12:54:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:40,686 INFO [aba5a916dfea:36083 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T12:54:40,688 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=OPENING, regionLocation=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:40,695 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, ASSIGN because future has completed 2024-11-19T12:54:40,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058}] 2024-11-19T12:54:40,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:40,854 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,854 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cfd198455e201428133912868c9b07e1, NAME => 'testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:40,855 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732020880447 cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,855 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:40,855 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,855 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,857 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,862 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfd198455e201428133912868c9b07e1 columnFamilyName family 2024-11-19T12:54:40,862 DEBUG [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:40,863 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(327): Store=cfd198455e201428133912868c9b07e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:40,863 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,864 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,865 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,865 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,865 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,866 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,873 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:40,873 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cfd198455e201428133912868c9b07e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67210602, jitterRate=0.0015160143375396729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:54:40,874 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:40,875 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cfd198455e201428133912868c9b07e1: Running coprocessor pre-open hook at 1732020880856Writing region info on filesystem at 1732020880856Initializing all the Stores at 1732020880856Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020880856Cleaning up temporary data from old regions at 1732020880865 (+9 ms)Running coprocessor post-open hooks at 1732020880874 (+9 ms)Region opened successfully at 1732020880875 (+1 ms) 2024-11-19T12:54:40,876 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1., pid=6, masterSystemTime=1732020880850 2024-11-19T12:54:40,879 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,880 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:40,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:40,884 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058 because future has completed 2024-11-19T12:54:40,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T12:54:40,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058 in 189 msec 2024-11-19T12:54:40,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T12:54:40,890 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, ASSIGN in 360 msec 2024-11-19T12:54:40,891 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:54:40,891 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732020880447","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020880891"}]},"ts":"1732020880891"} 2024-11-19T12:54:40,894 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732020880447, state=ENABLED in hbase:meta 2024-11-19T12:54:40,895 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:54:40,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732020880447 in 420 msec 2024-11-19T12:54:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:41,121 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732020880447 completed 2024-11-19T12:54:41,122 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732020880447 get assigned. Timeout = 60000ms 2024-11-19T12:54:41,125 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:54:41,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732020880447 assigned to meta. Checking AM states. 2024-11-19T12:54:41,131 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:54:41,131 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732020880447 assigned. 2024-11-19T12:54:41,140 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732020880447', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1., hostname=aba5a916dfea,37283,1732020878058, seqNum=2] 2024-11-19T12:54:41,152 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfd198455e201428133912868c9b07e1 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T12:54:41,175 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:41,210 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/.tmp/family/c8c9460154f2432b9bce472517bb2db7 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T12:54:41,214 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/.tmp/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:41,246 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/.tmp/family/c8c9460154f2432b9bce472517bb2db7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:41,247 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, delay=955 ms 2024-11-19T12:54:41,249 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T12:54:41,255 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for cfd198455e201428133912868c9b07e1 in 103ms, sequenceid=6, compaction requested=false 2024-11-19T12:54:41,256 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732020880447' 2024-11-19T12:54:41,258 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cfd198455e201428133912868c9b07e1: 2024-11-19T12:54:42,206 DEBUG [hfile-prefetch-1732020881247 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c8c9460154f2432b9bce472517bb2db7_0. 2024-11-19T12:54:42,211 DEBUG [hfile-prefetch-1732020881247 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c8c9460154f2432b9bce472517bb2db7_117. 2024-11-19T12:54:42,212 DEBUG [hfile-prefetch-1732020881247 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, with total blocks 2, and data blocks 1 2024-11-19T12:54:42,212 DEBUG [hfile-prefetch-1732020881247 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T12:54:42,213 DEBUG [hfile-prefetch-1732020881247 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T12:54:42,213 DEBUG [hfile-prefetch-1732020881247 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7. Will sleep 100ms and try the verification again. 2024-11-19T12:54:42,272 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=cfd198455e201428133912868c9b07e1, source=aba5a916dfea,37283,1732020878058, destination=aba5a916dfea,44249,1732020878208, warming up region on aba5a916dfea,44249,1732020878208 2024-11-19T12:54:42,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:54:42,274 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=cfd198455e201428133912868c9b07e1, source=aba5a916dfea,37283,1732020878058, destination=aba5a916dfea,44249,1732020878208, running balancer 2024-11-19T12:54:42,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE 2024-11-19T12:54:42,275 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE 2024-11-19T12:54:42,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:54:42,277 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=CLOSING, regionLocation=aba5a916dfea,37283,1732020878058 2024-11-19T12:54:42,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(7855): Warmup {ENCODED => cfd198455e201428133912868c9b07e1, NAME => 'testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:42,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:42,278 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,279 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfd198455e201428133912868c9b07e1 columnFamilyName family 2024-11-19T12:54:42,279 DEBUG [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:42,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE because future has completed 2024-11-19T12:54:42,281 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-19T12:54:42,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058}] 2024-11-19T12:54:42,283 DEBUG [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:42,284 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(327): Store=cfd198455e201428133912868c9b07e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1722): Closing cfd198455e201428133912868c9b07e1, disabling compactions & flushes 2024-11-19T12:54:42,284 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. after waiting 0 ms 2024-11-19T12:54:42,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,285 DEBUG [StoreFileCloser-cfd198455e201428133912868c9b07e1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:42,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44249 {}] regionserver.HRegion(1676): Region close journal for cfd198455e201428133912868c9b07e1: Waiting for close lock at 1732020882284Disabling compacts and flushes for region at 1732020882284Disabling writes for close at 1732020882284Writing region close event to WAL at 1732020882286 (+2 ms)Closed at 1732020882286 2024-11-19T12:54:42,288 WARN [hfile-prefetch-1732020881247 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-19T12:54:42,288 DEBUG [hfile-prefetch-1732020881247 {}] hfile.PrefetchExecutor(121): Prefetch completed for c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36083 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-19T12:54:42,351 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T12:54:42,351 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T12:54:42,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,359 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T12:54:42,359 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T12:54:42,360 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,360 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:42,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020882351 2024-11-19T12:54:42,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-19T12:54:42,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732020882359 2024-11-19T12:54:42,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,439 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,440 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-19T12:54:42,441 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing cfd198455e201428133912868c9b07e1, disabling compactions & flushes 2024-11-19T12:54:42,441 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,441 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,441 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. after waiting 0 ms 2024-11-19T12:54:42,441 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,443 DEBUG [StoreFileCloser-cfd198455e201428133912868c9b07e1-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file c8c9460154f2432b9bce472517bb2db7, starting offset: 0, end offset: 9223372036854775807 2024-11-19T12:54:42,445 DEBUG [StoreFileCloser-cfd198455e201428133912868c9b07e1-family-1 {}] bucket.BucketCache(856): removed key c8c9460154f2432b9bce472517bb2db7_0 from back map with offset lock 2094080 in the evict process 2024-11-19T12:54:42,450 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T12:54:42,450 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,451 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for cfd198455e201428133912868c9b07e1: Waiting for close lock at 1732020882441Running coprocessor pre-close hooks at 1732020882441Disabling compacts and flushes for region at 1732020882441Disabling writes for close at 1732020882441Writing region close event to WAL at 1732020882446 (+5 ms)Running coprocessor post-close hooks at 1732020882450 (+4 ms)Closed at 1732020882450 2024-11-19T12:54:42,451 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding cfd198455e201428133912868c9b07e1 move to aba5a916dfea,44249,1732020878208 record at close sequenceid=6 2024-11-19T12:54:42,454 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,454 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=CLOSED 2024-11-19T12:54:42,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058 because future has completed 2024-11-19T12:54:42,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T12:54:42,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,37283,1732020878058 in 177 msec 2024-11-19T12:54:42,461 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE; state=CLOSED, location=aba5a916dfea,44249,1732020878208; forceNewPlan=false, retain=false 2024-11-19T12:54:42,470 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T12:54:42,470 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T12:54:42,470 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,470 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,473 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T12:54:42,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T12:54:42,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,474 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:42,474 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020882470 2024-11-19T12:54:42,474 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,476 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:42,476 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020882473 2024-11-19T12:54:42,476 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,612 INFO [aba5a916dfea:36083 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T12:54:42,613 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=OPENING, regionLocation=aba5a916dfea,44249,1732020878208 2024-11-19T12:54:42,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE because future has completed 2024-11-19T12:54:42,619 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,44249,1732020878208}] 2024-11-19T12:54:42,776 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,777 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => cfd198455e201428133912868c9b07e1, NAME => 'testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:42,778 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732020880447 cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,778 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:42,778 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,778 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,780 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,781 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfd198455e201428133912868c9b07e1 columnFamilyName family 2024-11-19T12:54:42,782 DEBUG [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:42,784 DEBUG [StoreFileOpener-cfd198455e201428133912868c9b07e1-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7, delay=983 ms 2024-11-19T12:54:42,785 DEBUG [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:42,786 INFO [StoreOpener-cfd198455e201428133912868c9b07e1-1 {}] regionserver.HStore(327): Store=cfd198455e201428133912868c9b07e1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:42,786 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,787 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,788 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,792 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,793 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened cfd198455e201428133912868c9b07e1; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61797982, jitterRate=-0.07913830876350403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:54:42,793 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,795 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for cfd198455e201428133912868c9b07e1: Running coprocessor pre-open hook at 1732020882779Writing region info on filesystem at 1732020882779Initializing all the Stores at 1732020882779Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020882779Cleaning up temporary data from old regions at 1732020882788 (+9 ms)Running coprocessor post-open hooks at 1732020882793 (+5 ms)Region opened successfully at 1732020882794 (+1 ms) 2024-11-19T12:54:42,796 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1., pid=9, masterSystemTime=1732020882772 2024-11-19T12:54:42,799 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,800 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cfd198455e201428133912868c9b07e1, regionState=OPEN, openSeqNum=10, regionLocation=aba5a916dfea,44249,1732020878208 2024-11-19T12:54:42,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,44249,1732020878208 because future has completed 2024-11-19T12:54:42,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-19T12:54:42,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure cfd198455e201428133912868c9b07e1, server=aba5a916dfea,44249,1732020878208 in 187 msec 2024-11-19T12:54:42,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732020880447, region=cfd198455e201428133912868c9b07e1, REOPEN/MOVE in 534 msec 2024-11-19T12:54:42,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T12:54:42,828 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:42,829 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:42,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:42,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:42,833 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:42,833 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T12:54:42,833 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-11-19T12:54:42,834 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=aba5a916dfea,36083,1732020877205 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:42,856 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:42,856 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:42,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:42,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:42,858 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:42,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:42,858 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:42,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:42,860 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,37283,1732020878058' ***** 2024-11-19T12:54:42,860 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:54:42,860 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,44249,1732020878208' ***** 2024-11-19T12:54:42,860 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:54:42,860 INFO [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T12:54:42,860 INFO [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T12:54:42,861 INFO [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence 2024-11-19T12:54:42,861 INFO [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/bucket.persistence 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T12:54:42,861 INFO [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T12:54:42,861 INFO [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T12:54:42,862 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T12:54:42,862 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T12:54:42,872 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1487): Thread RS:1;aba5a916dfea:44249 started persisting bucket cache to file 2024-11-19T12:54:42,872 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1487): Thread RS:0;aba5a916dfea:37283 started persisting bucket cache to file 2024-11-19T12:54:42,872 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,872 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:42,872 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,872 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:42,874 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:42,874 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:42,875 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020882872 2024-11-19T12:54:42,875 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020882872 2024-11-19T12:54:42,875 DEBUG [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1509): Thread RS:1;aba5a916dfea:44249 finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,875 DEBUG [RS:0;aba5a916dfea:37283 {}] bucket.BucketCache(1509): Thread RS:0;aba5a916dfea:37283 finished persisting bucket cache to file, renaming 2024-11-19T12:54:42,875 WARN [RS:1;aba5a916dfea:44249 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-19T12:54:42,875 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:42,875 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:42,875 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:42,875 INFO [RS:0;aba5a916dfea:37283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:42,875 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:42,875 INFO [RS:1;aba5a916dfea:44249 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:42,875 INFO [RS:0;aba5a916dfea:37283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:42,875 INFO [RS:1;aba5a916dfea:44249 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:42,875 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,37283,1732020878058 2024-11-19T12:54:42,875 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(3091): Received CLOSE for cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,875 INFO [RS:0;aba5a916dfea:37283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:42,876 INFO [RS:0;aba5a916dfea:37283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;aba5a916dfea:37283. 2024-11-19T12:54:42,876 DEBUG [RS:0;aba5a916dfea:37283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:42,876 DEBUG [RS:0;aba5a916dfea:37283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:42,876 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:42,876 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:42,876 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:42,876 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,44249,1732020878208 2024-11-19T12:54:42,876 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T12:54:42,876 INFO [RS:1;aba5a916dfea:44249 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:42,876 INFO [RS:1;aba5a916dfea:44249 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;aba5a916dfea:44249. 2024-11-19T12:54:42,876 DEBUG [RS:1;aba5a916dfea:44249 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:42,876 DEBUG [RS:1;aba5a916dfea:44249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:42,877 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T12:54:42,877 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T12:54:42,877 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1325): Online Regions={cfd198455e201428133912868c9b07e1=testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1.} 2024-11-19T12:54:42,877 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T12:54:42,877 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T12:54:42,877 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cfd198455e201428133912868c9b07e1, disabling compactions & flushes 2024-11-19T12:54:42,877 DEBUG [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T12:54:42,877 DEBUG [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1351): Waiting on cfd198455e201428133912868c9b07e1 2024-11-19T12:54:42,877 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,877 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. after waiting 0 ms 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:54:42,877 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:54:42,877 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-19T12:54:42,878 DEBUG [StoreFileCloser-cfd198455e201428133912868c9b07e1-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/family/c8c9460154f2432b9bce472517bb2db7 2024-11-19T12:54:42,882 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnRegionMove1732020880447/cfd198455e201428133912868c9b07e1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T12:54:42,882 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,882 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cfd198455e201428133912868c9b07e1: Waiting for close lock at 1732020882876Running coprocessor pre-close hooks at 1732020882877 (+1 ms)Disabling compacts and flushes for region at 1732020882877Disabling writes for close at 1732020882877Writing region close event to WAL at 1732020882878 (+1 ms)Running coprocessor post-close hooks at 1732020882882 (+4 ms)Closed at 1732020882882 2024-11-19T12:54:42,882 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1. 2024-11-19T12:54:42,886 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 938c8adb8ea846dbb582dfcc194255ff 2024-11-19T12:54:42,893 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:42,896 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:42,903 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/938c8adb8ea846dbb582dfcc194255ff is 205, key is testBlockEvictionOnRegionMove1732020880447,,1732020880468.cfd198455e201428133912868c9b07e1./info:regioninfo/1732020882800/Put/seqid=0 2024-11-19T12:54:42,904 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/938c8adb8ea846dbb582dfcc194255ff 2024-11-19T12:54:42,913 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d6fcaec75cc349d391e313ac7de24d72 2024-11-19T12:54:42,925 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/d6fcaec75cc349d391e313ac7de24d72 is 43, key is default/ns:d/1732020880281/Put/seqid=0 2024-11-19T12:54:42,926 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/d6fcaec75cc349d391e313ac7de24d72 2024-11-19T12:54:42,934 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad38033fb20c46de9c51903188f67b9d 2024-11-19T12:54:42,947 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/ad38033fb20c46de9c51903188f67b9d is 78, key is testBlockEvictionOnRegionMove1732020880447/table:state/1732020880891/Put/seqid=0 2024-11-19T12:54:42,948 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/ad38033fb20c46de9c51903188f67b9d 2024-11-19T12:54:42,951 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/938c8adb8ea846dbb582dfcc194255ff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/938c8adb8ea846dbb582dfcc194255ff 2024-11-19T12:54:42,953 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/938c8adb8ea846dbb582dfcc194255ff, delay=968 ms 2024-11-19T12:54:42,954 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/938c8adb8ea846dbb582dfcc194255ff, entries=15, sequenceid=15, filesize=7.6 K 2024-11-19T12:54:42,955 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/d6fcaec75cc349d391e313ac7de24d72 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/d6fcaec75cc349d391e313ac7de24d72 2024-11-19T12:54:42,956 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/d6fcaec75cc349d391e313ac7de24d72, delay=976 ms 2024-11-19T12:54:42,958 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/d6fcaec75cc349d391e313ac7de24d72, entries=2, sequenceid=15, filesize=5.0 K 2024-11-19T12:54:42,958 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/ad38033fb20c46de9c51903188f67b9d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/ad38033fb20c46de9c51903188f67b9d 2024-11-19T12:54:42,959 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/ad38033fb20c46de9c51903188f67b9d, delay=965 ms 2024-11-19T12:54:42,960 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/ad38033fb20c46de9c51903188f67b9d, entries=2, sequenceid=15, filesize=5.3 K 2024-11-19T12:54:42,961 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false 2024-11-19T12:54:42,961 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T12:54:42,963 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/938c8adb8ea846dbb582dfcc194255ff 2024-11-19T12:54:42,965 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/d6fcaec75cc349d391e313ac7de24d72 2024-11-19T12:54:42,965 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/ad38033fb20c46de9c51903188f67b9d 2024-11-19T12:54:42,970 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-19T12:54:42,970 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:54:42,970 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:42,971 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020882877Running coprocessor pre-close hooks at 1732020882877Disabling compacts and flushes for region at 1732020882877Disabling writes for close at 1732020882877Obtaining lock to block concurrent updates at 1732020882878 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732020882878Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732020882878Flushing stores of hbase:meta,,1.1588230740 at 1732020882878Flushing 1588230740/info: creating writer at 1732020882878Flushing 1588230740/info: appending metadata at 1732020882902 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732020882902Flushing 1588230740/ns: creating writer at 1732020882907 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732020882924 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732020882924Flushing 1588230740/table: creating writer at 1732020882929 (+5 ms)Flushing 1588230740/table: appending metadata at 1732020882946 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732020882946Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f538c53: reopening flushed file at 1732020882951 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45148ffe: reopening flushed file at 1732020882954 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22431bd0: reopening flushed file at 1732020882958 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false at 1732020882961 (+3 ms)Writing region close event to WAL at 1732020882966 (+5 ms)Running coprocessor post-close hooks at 1732020882970 (+4 ms)Closed at 1732020882970 2024-11-19T12:54:42,971 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:43,078 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,44249,1732020878208; all regions closed. 2024-11-19T12:54:43,078 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,37283,1732020878058; all regions closed. 2024-11-19T12:54:43,088 DEBUG [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:43,088 DEBUG [RS:1;aba5a916dfea:44249 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:43,088 INFO [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C37283%2C1732020878058.meta:.meta(num 1732020880135) 2024-11-19T12:54:43,088 INFO [RS:1;aba5a916dfea:44249 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C44249%2C1732020878208:(num 1732020879749) 2024-11-19T12:54:43,088 DEBUG [RS:1;aba5a916dfea:44249 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:43,089 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:43,089 INFO [RS:1;aba5a916dfea:44249 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:43,089 INFO [RS:1;aba5a916dfea:44249 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:43,089 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:43,090 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:43,090 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:43,090 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:43,090 INFO [RS:1;aba5a916dfea:44249 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:43,090 INFO [RS:1;aba5a916dfea:44249 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44249 2024-11-19T12:54:43,091 DEBUG [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:43,091 INFO [RS:0;aba5a916dfea:37283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C37283%2C1732020878058:(num 1732020879749) 2024-11-19T12:54:43,091 DEBUG [RS:0;aba5a916dfea:37283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:43,091 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:43,091 INFO [RS:0;aba5a916dfea:37283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:43,092 INFO [RS:0;aba5a916dfea:37283 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:43,092 INFO [RS:0;aba5a916dfea:37283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:43,092 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:43,092 INFO [RS:0;aba5a916dfea:37283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37283 2024-11-19T12:54:43,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:43,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,44249,1732020878208 2024-11-19T12:54:43,117 INFO [RS:1;aba5a916dfea:44249 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:43,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,37283,1732020878058 2024-11-19T12:54:43,176 INFO [RS:0;aba5a916dfea:37283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:43,184 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,37283,1732020878058] 2024-11-19T12:54:43,200 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,37283,1732020878058 already deleted, retry=false 2024-11-19T12:54:43,201 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aba5a916dfea,37283,1732020878058 expired; onlineServers=1 2024-11-19T12:54:43,201 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,44249,1732020878208] 2024-11-19T12:54:43,209 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,44249,1732020878208 already deleted, retry=false 2024-11-19T12:54:43,209 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aba5a916dfea,44249,1732020878208 expired; onlineServers=0 2024-11-19T12:54:43,209 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'aba5a916dfea,36083,1732020877205' ***** 2024-11-19T12:54:43,209 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T12:54:43,210 INFO [M:0;aba5a916dfea:36083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:43,210 INFO [M:0;aba5a916dfea:36083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:43,210 DEBUG [M:0;aba5a916dfea:36083 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T12:54:43,211 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T12:54:43,211 DEBUG [M:0;aba5a916dfea:36083 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T12:54:43,211 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020879371 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020879371,5,FailOnTimeoutGroup] 2024-11-19T12:54:43,211 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020879370 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020879370,5,FailOnTimeoutGroup] 2024-11-19T12:54:43,212 INFO [M:0;aba5a916dfea:36083 {}] hbase.ChoreService(370): Chore service for: master/aba5a916dfea:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:43,212 INFO [M:0;aba5a916dfea:36083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:43,212 DEBUG [M:0;aba5a916dfea:36083 {}] master.HMaster(1795): Stopping service threads 2024-11-19T12:54:43,212 INFO [M:0;aba5a916dfea:36083 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T12:54:43,213 INFO [M:0;aba5a916dfea:36083 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:43,214 INFO [M:0;aba5a916dfea:36083 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:54:43,214 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T12:54:43,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:43,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,223 DEBUG [M:0;aba5a916dfea:36083 {}] zookeeper.ZKUtil(347): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T12:54:43,223 WARN [M:0;aba5a916dfea:36083 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T12:54:43,223 INFO [M:0;aba5a916dfea:36083 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.lastflushedseqids 2024-11-19T12:54:43,235 INFO [M:0;aba5a916dfea:36083 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:54:43,235 INFO [M:0;aba5a916dfea:36083 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T12:54:43,235 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:54:43,235 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,235 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,235 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:54:43,235 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,236 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-19T12:54:43,243 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 194641bac39a4022917c3d2f70aa1a9c 2024-11-19T12:54:43,255 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/194641bac39a4022917c3d2f70aa1a9c is 82, key is hbase:meta,,1/info:regioninfo/1732020880194/Put/seqid=0 2024-11-19T12:54:43,256 INFO [M:0;aba5a916dfea:36083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/194641bac39a4022917c3d2f70aa1a9c 2024-11-19T12:54:43,263 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e649dcbd41554b9686b200d0b409cd77 2024-11-19T12:54:43,276 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e649dcbd41554b9686b200d0b409cd77 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732020880897/Put/seqid=0 2024-11-19T12:54:43,277 INFO [M:0;aba5a916dfea:36083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e649dcbd41554b9686b200d0b409cd77 2024-11-19T12:54:43,283 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3e7766c2418643efbdfe0fb994c23fc6 2024-11-19T12:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44249-0x1015472c93e0002, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,285 INFO [RS:1;aba5a916dfea:44249 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:43,285 INFO [RS:1;aba5a916dfea:44249 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,44249,1732020878208; zookeeper connection closed. 2024-11-19T12:54:43,285 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47bf73a8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47bf73a8 2024-11-19T12:54:43,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,292 INFO [RS:0;aba5a916dfea:37283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:43,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37283-0x1015472c93e0001, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,292 INFO [RS:0;aba5a916dfea:37283 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,37283,1732020878058; zookeeper connection closed. 2024-11-19T12:54:43,293 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a82f25b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a82f25b 2024-11-19T12:54:43,293 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-19T12:54:43,295 DEBUG [M:0;aba5a916dfea:36083 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7766c2418643efbdfe0fb994c23fc6 is 69, key is aba5a916dfea,37283,1732020878058/rs:state/1732020879455/Put/seqid=0 2024-11-19T12:54:43,295 INFO [M:0;aba5a916dfea:36083 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7766c2418643efbdfe0fb994c23fc6 2024-11-19T12:54:43,297 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/194641bac39a4022917c3d2f70aa1a9c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/194641bac39a4022917c3d2f70aa1a9c 2024-11-19T12:54:43,298 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/194641bac39a4022917c3d2f70aa1a9c, entries=8, sequenceid=84, filesize=5.5 K 2024-11-19T12:54:43,299 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e649dcbd41554b9686b200d0b409cd77 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e649dcbd41554b9686b200d0b409cd77 2024-11-19T12:54:43,300 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e649dcbd41554b9686b200d0b409cd77, entries=9, sequenceid=84, filesize=6.5 K 2024-11-19T12:54:43,300 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3e7766c2418643efbdfe0fb994c23fc6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3e7766c2418643efbdfe0fb994c23fc6 2024-11-19T12:54:43,302 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3e7766c2418643efbdfe0fb994c23fc6, entries=2, sequenceid=84, filesize=5.1 K 2024-11-19T12:54:43,302 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=84, compaction requested=false 2024-11-19T12:54:43,304 INFO [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,304 DEBUG [M:0;aba5a916dfea:36083 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020883235Disabling compacts and flushes for region at 1732020883235Disabling writes for close at 1732020883235Obtaining lock to block concurrent updates at 1732020883236 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732020883236Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732020883236Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732020883236Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732020883236Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732020883255 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732020883255Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732020883258 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732020883275 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732020883275Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732020883278 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732020883294 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732020883294Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffd2fed: reopening flushed file at 1732020883297 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e0cbafd: reopening flushed file at 1732020883298 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65d35a8: reopening flushed file at 1732020883300 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=84, compaction requested=false at 1732020883302 (+2 ms)Writing region close event to WAL at 1732020883304 (+2 ms)Closed at 1732020883304 2024-11-19T12:54:43,306 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:43,306 INFO [M:0;aba5a916dfea:36083 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T12:54:43,306 INFO [M:0;aba5a916dfea:36083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36083 2024-11-19T12:54:43,306 INFO [M:0;aba5a916dfea:36083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:43,456 INFO [M:0;aba5a916dfea:36083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:43,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36083-0x1015472c93e0000, quorum=127.0.0.1:65054, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:43,463 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T12:54:43,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T12:54:43,484 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,491 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/aba5a916dfea:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732020882953 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732020881247 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/aba5a916dfea:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/aba5a916dfea:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732020882956 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732020882784 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/aba5a916dfea:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 502), ProcessCount=11 (was 11), AvailableMemoryMB=6852 (was 7235) 2024-11-19T12:54:43,496 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=478, ProcessCount=11, AvailableMemoryMB=6852 2024-11-19T12:54:43,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/hadoop-log-dir so I do NOT create it in target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5 2024-11-19T12:54:43,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/hadoop-tmp-dir so I do NOT create it in target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5 2024-11-19T12:54:43,497 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,499 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/cluster_fe3299fb-35ae-5410-52b7-ec4eddbb8d4e, deleteOnExit=true 2024-11-19T12:54:43,502 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/cluster_fe3299fb-35ae-5410-52b7-ec4eddbb8d4e/zookeeper_0, clientPort=52121, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/cluster_fe3299fb-35ae-5410-52b7-ec4eddbb8d4e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/cluster_fe3299fb-35ae-5410-52b7-ec4eddbb8d4e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T12:54:43,503 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52121 2024-11-19T12:54:43,515 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 with version=8 2024-11-19T12:54:43,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase-staging 2024-11-19T12:54:43,518 INFO [Time-limited test {}] client.ConnectionUtils(128): master/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:43,518 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T12:54:43,519 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:43,519 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36439 2024-11-19T12:54:43,522 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36439 connecting to ZooKeeper ensemble=127.0.0.1:52121 2024-11-19T12:54:43,560 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364390x0, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:43,560 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36439-0x1015472e2f20000 connected 2024-11-19T12:54:43,644 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,645 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:43,647 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697, hbase.cluster.distributed=false 2024-11-19T12:54:43,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:43,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36439 2024-11-19T12:54:43,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36439 2024-11-19T12:54:43,651 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36439 2024-11-19T12:54:43,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36439 2024-11-19T12:54:43,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36439 2024-11-19T12:54:43,671 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:54:43,671 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:43,672 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44109 2024-11-19T12:54:43,673 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44109 connecting to ZooKeeper ensemble=127.0.0.1:52121 2024-11-19T12:54:43,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,674 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:441090x0, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:43,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44109-0x1015472e2f20001 connected 2024-11-19T12:54:43,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:43,686 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:54:43,687 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache 2024-11-19T12:54:43,687 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T12:54:43,687 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T12:54:43,687 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T12:54:43,687 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T12:54:43,687 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T12:54:43,688 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T12:54:43,688 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:54:43,688 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T12:54:43,688 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence is complete. 2024-11-19T12:54:43,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:54:43,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:43,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-19T12:54:43,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44109 2024-11-19T12:54:43,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44109 2024-11-19T12:54:43,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-19T12:54:43,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44109 2024-11-19T12:54:43,710 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:54:43,710 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:43,711 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36557 2024-11-19T12:54:43,712 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36557 connecting to ZooKeeper ensemble=127.0.0.1:52121 2024-11-19T12:54:43,713 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,713 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:365570x0, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:43,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:365570x0, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:43,725 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36557-0x1015472e2f20002 connected 2024-11-19T12:54:43,725 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:54:43,726 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache 2024-11-19T12:54:43,726 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T12:54:43,726 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T12:54:43,726 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T12:54:43,727 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T12:54:43,727 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T12:54:43,727 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T12:54:43,727 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence is complete. 2024-11-19T12:54:43,727 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T12:54:43,728 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:54:43,729 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T12:54:43,730 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:43,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36557 2024-11-19T12:54:43,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36557 2024-11-19T12:54:43,737 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36557 2024-11-19T12:54:43,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36557 2024-11-19T12:54:43,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36557 2024-11-19T12:54:43,749 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;aba5a916dfea:36439 2024-11-19T12:54:43,750 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/aba5a916dfea,36439,1732020883517 2024-11-19T12:54:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,761 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/aba5a916dfea,36439,1732020883517 2024-11-19T12:54:43,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:43,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:43,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,769 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T12:54:43,770 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/aba5a916dfea,36439,1732020883517 from backup master directory 2024-11-19T12:54:43,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/aba5a916dfea,36439,1732020883517 2024-11-19T12:54:43,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T12:54:43,777 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:43,777 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=aba5a916dfea,36439,1732020883517 2024-11-19T12:54:43,778 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase.id] with ID: fff98aaa-7ebe-4cd6-a594-2aa139895381 2024-11-19T12:54:43,778 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.tmp/hbase.id 2024-11-19T12:54:43,781 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/hbase.id] 2024-11-19T12:54:43,822 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:43,822 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T12:54:43,825 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-19T12:54:43,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:43,856 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:43,857 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T12:54:43,858 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:43,883 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store 2024-11-19T12:54:43,890 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:54:43,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020883891Disabling compacts and flushes for region at 1732020883891Disabling writes for close at 1732020883891Writing region close event to WAL at 1732020883891Closed at 1732020883891 2024-11-19T12:54:43,891 WARN [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/.initializing 2024-11-19T12:54:43,891 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36439,1732020883517 2024-11-19T12:54:43,901 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:43,904 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=aba5a916dfea%2C36439%2C1732020883517, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36439,1732020883517, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/oldWALs, maxLogs=10 2024-11-19T12:54:43,919 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/WALs/aba5a916dfea,36439,1732020883517/aba5a916dfea%2C36439%2C1732020883517.1732020883904 2024-11-19T12:54:43,919 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:43,919 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:43,920 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:43,920 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,920 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,921 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T12:54:43,926 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:43,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:43,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T12:54:43,930 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:43,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:43,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T12:54:43,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:43,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:43,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T12:54:43,938 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:43,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:43,938 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,939 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,939 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,939 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,939 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,940 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:43,940 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T12:54:43,946 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:43,946 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64646029, jitterRate=-0.03669910132884979}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:43,947 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732020883920Initializing all the Stores at 1732020883920Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020883920Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020883921 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020883921Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020883921Cleaning up temporary data from old regions at 1732020883939 (+18 ms)Region opened successfully at 1732020883947 (+8 ms) 2024-11-19T12:54:43,948 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T12:54:43,952 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e48b768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:43,953 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T12:54:43,953 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T12:54:43,953 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T12:54:43,953 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T12:54:43,954 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T12:54:43,954 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-19T12:54:43,954 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T12:54:43,956 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T12:54:43,957 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T12:54:43,977 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T12:54:43,977 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T12:54:43,978 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T12:54:43,985 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T12:54:43,985 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T12:54:43,986 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T12:54:43,993 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T12:54:43,995 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T12:54:44,002 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T12:54:44,004 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T12:54:44,016 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,028 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=aba5a916dfea,36439,1732020883517, sessionid=0x1015472e2f20000, setting cluster-up flag (Was=false) 2024-11-19T12:54:44,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,069 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T12:54:44,070 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36439,1732020883517 2024-11-19T12:54:44,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,110 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T12:54:44,111 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=aba5a916dfea,36439,1732020883517 2024-11-19T12:54:44,112 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T12:54:44,113 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:44,113 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T12:54:44,114 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T12:54:44,114 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: aba5a916dfea,36439,1732020883517 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/aba5a916dfea:0, corePoolSize=5, maxPoolSize=5 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/aba5a916dfea:0, corePoolSize=10, maxPoolSize=10 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:44,116 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732020914118 2024-11-19T12:54:44,118 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T12:54:44,118 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T12:54:44,118 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,118 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,119 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T12:54:44,119 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T12:54:44,119 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T12:54:44,119 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T12:54:44,119 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T12:54:44,119 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T12:54:44,120 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020884119,5,FailOnTimeoutGroup] 2024-11-19T12:54:44,120 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020884120,5,FailOnTimeoutGroup] 2024-11-19T12:54:44,120 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,120 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T12:54:44,120 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,120 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,129 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T12:54:44,129 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:44,135 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:44,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T12:54:44,140 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:54:44,140 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,140 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T12:54:44,141 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(746): ClusterId : fff98aaa-7ebe-4cd6-a594-2aa139895381 2024-11-19T12:54:44,141 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(746): ClusterId : fff98aaa-7ebe-4cd6-a594-2aa139895381 2024-11-19T12:54:44,141 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:54:44,141 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:54:44,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:54:44,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,145 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T12:54:44,150 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:54:44,150 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,150 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T12:54:44,153 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:54:44,153 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:54:44,153 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:54:44,153 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:54:44,155 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:54:44,155 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,155 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:44,156 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:54:44,161 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:54:44,161 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:54:44,162 DEBUG [RS:1;aba5a916dfea:36557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b5a3f9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:44,162 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:44,162 DEBUG [RS:0;aba5a916dfea:44109 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ab53b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:44,162 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59676263, jitterRate=-0.11075438559055328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:44,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732020884135Initializing all the Stores at 1732020884135Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884135Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884136 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020884136Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884136Cleaning up temporary data from old regions at 1732020884156 (+20 ms)Region opened successfully at 1732020884163 (+7 ms) 2024-11-19T12:54:44,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:54:44,163 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:54:44,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:54:44,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:54:44,163 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:54:44,164 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:44,164 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020884163Disabling compacts and flushes for region at 1732020884163Disabling writes for close at 1732020884163Writing region close event to WAL at 1732020884164 (+1 ms)Closed at 1732020884164 2024-11-19T12:54:44,165 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:44,165 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T12:54:44,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T12:54:44,167 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:54:44,167 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T12:54:44,174 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;aba5a916dfea:36557 2024-11-19T12:54:44,174 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:54:44,174 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;aba5a916dfea:44109 2024-11-19T12:54:44,174 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:54:44,174 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:54:44,174 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:54:44,174 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:54:44,174 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:54:44,175 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36439,1732020883517 with port=36557, startcode=1732020883709 2024-11-19T12:54:44,175 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36439,1732020883517 with port=44109, startcode=1732020883670 2024-11-19T12:54:44,175 DEBUG [RS:0;aba5a916dfea:44109 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:54:44,175 DEBUG [RS:1;aba5a916dfea:36557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:54:44,179 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38433, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:54:44,179 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:54:44,180 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,180 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,181 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,181 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,182 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:44,182 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T12:54:44,182 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:54:44,183 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:44,183 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T12:54:44,183 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:54:44,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:44,227 DEBUG [RS:0;aba5a916dfea:44109 {}] zookeeper.ZKUtil(111): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,227 WARN [RS:0;aba5a916dfea:44109 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:44,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,36557,1732020883709] 2024-11-19T12:54:44,227 INFO [RS:0;aba5a916dfea:44109 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:44,227 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,44109,1732020883670] 2024-11-19T12:54:44,227 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,227 DEBUG [RS:1;aba5a916dfea:36557 {}] zookeeper.ZKUtil(111): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,227 WARN [RS:1;aba5a916dfea:36557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:44,228 INFO [RS:1;aba5a916dfea:36557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:44,228 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,235 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:54:44,235 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:54:44,239 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:54:44,239 INFO [RS:0;aba5a916dfea:44109 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:54:44,239 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,240 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:54:44,240 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:54:44,241 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,241 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,241 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 INFO [RS:1;aba5a916dfea:36557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,242 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,242 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:44,243 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:54:44,243 DEBUG [RS:0;aba5a916dfea:44109 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:44,243 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:54:44,243 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,243 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,243 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,243 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,243 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,243 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,244 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,44109,1732020883670-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,244 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,245 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,245 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,245 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:44,245 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:44,245 DEBUG [RS:1;aba5a916dfea:36557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:44,245 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,246 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,246 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,246 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,246 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,246 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36557,1732020883709-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:44,260 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:54:44,260 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,44109,1732020883670-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,260 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,260 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.Replication(171): aba5a916dfea,44109,1732020883670 started 2024-11-19T12:54:44,260 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:54:44,261 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36557,1732020883709-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,261 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,261 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.Replication(171): aba5a916dfea,36557,1732020883709 started 2024-11-19T12:54:44,274 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,275 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,275 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,44109,1732020883670, RpcServer on aba5a916dfea/172.17.0.2:44109, sessionid=0x1015472e2f20001 2024-11-19T12:54:44,275 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,36557,1732020883709, RpcServer on aba5a916dfea/172.17.0.2:36557, sessionid=0x1015472e2f20002 2024-11-19T12:54:44,275 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:54:44,275 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:54:44,275 DEBUG [RS:1;aba5a916dfea:36557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,275 DEBUG [RS:0;aba5a916dfea:44109 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,275 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,36557,1732020883709' 2024-11-19T12:54:44,275 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,44109,1732020883670' 2024-11-19T12:54:44,275 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:54:44,275 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:54:44,276 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,276 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,36557,1732020883709' 2024-11-19T12:54:44,276 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:54:44,276 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:54:44,276 DEBUG [RS:0;aba5a916dfea:44109 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,44109,1732020883670 2024-11-19T12:54:44,276 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,44109,1732020883670' 2024-11-19T12:54:44,277 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:54:44,277 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:54:44,277 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:54:44,277 DEBUG [RS:1;aba5a916dfea:36557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:54:44,277 INFO [RS:1;aba5a916dfea:36557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:54:44,277 INFO [RS:1;aba5a916dfea:36557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:54:44,277 DEBUG [RS:0;aba5a916dfea:44109 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:54:44,277 INFO [RS:0;aba5a916dfea:44109 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:54:44,277 INFO [RS:0;aba5a916dfea:44109 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:54:44,318 WARN [aba5a916dfea:36439 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T12:54:44,378 INFO [RS:1;aba5a916dfea:36557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:44,379 INFO [RS:0;aba5a916dfea:44109 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:44,380 INFO [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C36557%2C1732020883709, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,36557,1732020883709, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:44,380 INFO [RS:0;aba5a916dfea:44109 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C44109%2C1732020883670, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:44,401 INFO [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,36557,1732020883709/aba5a916dfea%2C36557%2C1732020883709.1732020884384 2024-11-19T12:54:44,401 INFO [RS:0;aba5a916dfea:44109 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670/aba5a916dfea%2C44109%2C1732020883670.1732020884385 2024-11-19T12:54:44,401 DEBUG [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:44,402 DEBUG [RS:0;aba5a916dfea:44109 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:44,568 DEBUG [aba5a916dfea:36439 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T12:54:44,569 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(204): Hosts are {aba5a916dfea=0} racks are {/default-rack=0} 2024-11-19T12:54:44,571 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:54:44,571 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:54:44,571 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:54:44,571 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:54:44,571 INFO [aba5a916dfea:36439 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:54:44,571 INFO [aba5a916dfea:36439 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:54:44,571 DEBUG [aba5a916dfea:36439 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:54:44,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,574 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,36557,1732020883709, state=OPENING 2024-11-19T12:54:44,624 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T12:54:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:44,636 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T12:54:44,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,637 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,36557,1732020883709}] 2024-11-19T12:54:44,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,793 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:54:44,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38121, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:54:44,804 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T12:54:44,804 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:44,805 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T12:54:44,807 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C36557%2C1732020883709.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,36557,1732020883709, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:44,823 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,36557,1732020883709/aba5a916dfea%2C36557%2C1732020883709.meta.1732020884807.meta 2024-11-19T12:54:44,824 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:44,824 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:44,824 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T12:54:44,825 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T12:54:44,825 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T12:54:44,825 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T12:54:44,825 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:44,825 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T12:54:44,825 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T12:54:44,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T12:54:44,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T12:54:44,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T12:54:44,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T12:54:44,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T12:54:44,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T12:54:44,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T12:54:44,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T12:54:44,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,829 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T12:54:44,829 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T12:54:44,830 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:44,830 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740 2024-11-19T12:54:44,830 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T12:54:44,830 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T12:54:44,830 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T12:54:44,831 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T12:54:44,831 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60437626, jitterRate=-0.09940919280052185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T12:54:44,831 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T12:54:44,832 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732020884825Writing region info on filesystem at 1732020884825Initializing all the Stores at 1732020884825Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884826 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884826Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020884826Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732020884826Cleaning up temporary data from old regions at 1732020884830 (+4 ms)Running coprocessor post-open hooks at 1732020884831 (+1 ms)Region opened successfully at 1732020884832 (+1 ms) 2024-11-19T12:54:44,833 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732020884792 2024-11-19T12:54:44,835 DEBUG [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T12:54:44,835 INFO [RS_OPEN_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T12:54:44,836 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,837 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as aba5a916dfea,36557,1732020883709, state=OPEN 2024-11-19T12:54:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:44,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T12:54:44,868 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,868 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T12:54:44,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T12:54:44,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=aba5a916dfea,36557,1732020883709 in 231 msec 2024-11-19T12:54:44,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T12:54:44,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 707 msec 2024-11-19T12:54:44,875 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T12:54:44,876 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T12:54:44,880 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:44,880 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,36557,1732020883709, seqNum=-1] 2024-11-19T12:54:44,881 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:44,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:44,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 775 msec 2024-11-19T12:54:44,888 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732020884888, completionTime=-1 2024-11-19T12:54:44,889 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T12:54:44,889 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T12:54:44,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T12:54:44,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732020944891 2024-11-19T12:54:44,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732021004891 2024-11-19T12:54:44,891 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-aba5a916dfea:36439, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,892 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,895 DEBUG [master/aba5a916dfea:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.121sec 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T12:54:44,898 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:44,899 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T12:54:44,899 DEBUG [master/aba5a916dfea:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T12:54:44,899 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T12:54:44,900 INFO [master/aba5a916dfea:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,36439,1732020883517-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:44,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:44,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request aba5a916dfea,36439,-1 for getting cluster id 2024-11-19T12:54:44,942 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T12:54:44,944 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fff98aaa-7ebe-4cd6-a594-2aa139895381' 2024-11-19T12:54:44,945 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T12:54:44,945 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fff98aaa-7ebe-4cd6-a594-2aa139895381" 2024-11-19T12:54:44,946 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49ae09ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:44,946 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [aba5a916dfea,36439,-1] 2024-11-19T12:54:44,946 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T12:54:44,947 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:44,950 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T12:54:44,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T12:54:44,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T12:54:44,954 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=aba5a916dfea,36557,1732020883709, seqNum=-1] 2024-11-19T12:54:44,954 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:44,957 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60356, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:44,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=aba5a916dfea,36439,1732020883517 2024-11-19T12:54:44,960 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T12:54:44,961 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is aba5a916dfea,36439,1732020883517 2024-11-19T12:54:44,961 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33d8c5a9 2024-11-19T12:54:44,961 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T12:54:44,963 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46030, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T12:54:44,964 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732020884959', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T12:54:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 2024-11-19T12:54:44,967 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T12:54:44,967 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:44,967 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732020884959" procId is: 4 2024-11-19T12:54:44,968 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T12:54:44,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:44,984 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1b95c7ead1b6f1bb58c55808f92c9fab, NAME => 'testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732020884959', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:44,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:44,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1722): Closing 1b95c7ead1b6f1bb58c55808f92c9fab, disabling compactions & flushes 2024-11-19T12:54:44,990 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:44,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:44,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. after waiting 0 ms 2024-11-19T12:54:44,990 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:44,990 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:44,991 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732020884959-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1b95c7ead1b6f1bb58c55808f92c9fab: Waiting for close lock at 1732020884990Disabling compacts and flushes for region at 1732020884990Disabling writes for close at 1732020884990Writing region close event to WAL at 1732020884990Closed at 1732020884990 2024-11-19T12:54:44,992 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T12:54:44,992 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732020884992"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732020884992"}]},"ts":"1732020884992"} 2024-11-19T12:54:44,994 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T12:54:44,995 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T12:54:44,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732020884959","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020884995"}]},"ts":"1732020884995"} 2024-11-19T12:54:44,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732020884959, state=ENABLING in hbase:meta 2024-11-19T12:54:44,998 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {aba5a916dfea=0} racks are {/default-rack=0} 2024-11-19T12:54:44,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T12:54:44,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T12:54:44,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T12:54:44,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T12:54:44,999 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T12:54:44,999 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T12:54:44,999 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T12:54:44,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN}] 2024-11-19T12:54:45,000 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN 2024-11-19T12:54:45,001 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN; state=OFFLINE, location=aba5a916dfea,44109,1732020883670; forceNewPlan=false, retain=false 2024-11-19T12:54:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:45,152 INFO [aba5a916dfea:36439 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T12:54:45,153 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b95c7ead1b6f1bb58c55808f92c9fab, regionState=OPENING, regionLocation=aba5a916dfea,44109,1732020883670 2024-11-19T12:54:45,157 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN because future has completed 2024-11-19T12:54:45,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,44109,1732020883670}] 2024-11-19T12:54:45,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:45,312 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T12:54:45,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42227, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T12:54:45,316 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:45,316 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b95c7ead1b6f1bb58c55808f92c9fab, NAME => 'testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:45,317 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732020884959 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,317 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:45,317 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,317 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,318 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,321 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b95c7ead1b6f1bb58c55808f92c9fab columnFamilyName family 2024-11-19T12:54:45,321 DEBUG [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:45,321 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] regionserver.HStore(327): Store=1b95c7ead1b6f1bb58c55808f92c9fab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:45,321 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,322 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,322 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,322 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,322 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,323 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,331 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T12:54:45,331 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1b95c7ead1b6f1bb58c55808f92c9fab; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60659549, jitterRate=-0.0961022824048996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:54:45,332 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:45,333 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1b95c7ead1b6f1bb58c55808f92c9fab: Running coprocessor pre-open hook at 1732020885317Writing region info on filesystem at 1732020885317Initializing all the Stores at 1732020885317Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020885317Cleaning up temporary data from old regions at 1732020885322 (+5 ms)Running coprocessor post-open hooks at 1732020885332 (+10 ms)Region opened successfully at 1732020885333 (+1 ms) 2024-11-19T12:54:45,334 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab., pid=6, masterSystemTime=1732020885312 2024-11-19T12:54:45,337 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:45,337 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:45,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1b95c7ead1b6f1bb58c55808f92c9fab, regionState=OPEN, openSeqNum=2, regionLocation=aba5a916dfea,44109,1732020883670 2024-11-19T12:54:45,341 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,44109,1732020883670 because future has completed 2024-11-19T12:54:45,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T12:54:45,344 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,44109,1732020883670 in 184 msec 2024-11-19T12:54:45,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T12:54:45,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN in 345 msec 2024-11-19T12:54:45,347 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T12:54:45,347 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732020884959","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732020885347"}]},"ts":"1732020885347"} 2024-11-19T12:54:45,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732020884959, state=ENABLED in hbase:meta 2024-11-19T12:54:45,350 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T12:54:45,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732020884959 in 386 msec 2024-11-19T12:54:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36439 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T12:54:45,601 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732020884959 completed 2024-11-19T12:54:45,601 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732020884959 get assigned. Timeout = 60000ms 2024-11-19T12:54:45,602 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:54:45,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732020884959 assigned to meta. Checking AM states. 2024-11-19T12:54:45,608 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T12:54:45,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732020884959 assigned. 2024-11-19T12:54:45,613 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732020884959', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab., hostname=aba5a916dfea,44109,1732020883670, seqNum=2] 2024-11-19T12:54:45,614 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T12:54:45,616 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T12:54:45,619 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1b95c7ead1b6f1bb58c55808f92c9fab 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T12:54:45,627 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:45,638 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/.tmp/family/bb409d63abed430c92dfb6a31fc369de is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T12:54:45,639 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/.tmp/family/bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:45,642 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/.tmp/family/bb409d63abed430c92dfb6a31fc369de as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:45,643 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, delay=970 ms 2024-11-19T12:54:45,644 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T12:54:45,644 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1b95c7ead1b6f1bb58c55808f92c9fab in 25ms, sequenceid=6, compaction requested=false 2024-11-19T12:54:45,644 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732020884959' 2024-11-19T12:54:45,645 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1b95c7ead1b6f1bb58c55808f92c9fab: 2024-11-19T12:54:45,669 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T12:54:46,613 DEBUG [hfile-prefetch-1732020882784 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bb409d63abed430c92dfb6a31fc369de_0. 2024-11-19T12:54:46,614 DEBUG [hfile-prefetch-1732020882784 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key bb409d63abed430c92dfb6a31fc369de_117. 2024-11-19T12:54:46,614 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, with total blocks 2, and data blocks 1 2024-11-19T12:54:46,614 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T12:54:46,614 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T12:54:46,614 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de. Will sleep 100ms and try the verification again. 2024-11-19T12:54:46,646 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;aba5a916dfea:44109,5,FailOnTimeoutGroup] 2024-11-19T12:54:46,646 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,44109,1732020883670' ***** 2024-11-19T12:54:46,646 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-19T12:54:46,646 INFO [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T12:54:46,646 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:46,702 INFO [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence 2024-11-19T12:54:46,702 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T12:54:46,702 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T12:54:46,702 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T12:54:46,703 INFO [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T12:54:46,703 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T12:54:46,713 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1487): Thread RS:0;aba5a916dfea:44109 started persisting bucket cache to file 2024-11-19T12:54:46,713 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:46,713 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:46,715 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, with total blocks 2, and data blocks 1 2024-11-19T12:54:46,715 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-19T12:54:46,715 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2293): found block for file bb409d63abed430c92dfb6a31fc369de in the backing map. Acquiring read lock for offset 0 2024-11-19T12:54:46,715 DEBUG [hfile-prefetch-1732020882784 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de has now been fully cached. 2024-11-19T12:54:46,715 DEBUG [hfile-prefetch-1732020882784 {}] hfile.PrefetchExecutor(121): Prefetch completed for bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:46,718 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-19T12:54:46,719 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732020886713 2024-11-19T12:54:46,719 DEBUG [RS:0;aba5a916dfea:44109 {}] bucket.BucketCache(1509): Thread RS:0;aba5a916dfea:44109 finished persisting bucket cache to file, renaming 2024-11-19T12:54:46,719 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:46,719 INFO [RS:0;aba5a916dfea:44109 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:46,719 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:46,719 INFO [RS:0;aba5a916dfea:44109 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:46,720 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:46,720 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,44109,1732020883670 2024-11-19T12:54:46,720 INFO [RS:0;aba5a916dfea:44109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:46,721 INFO [RS:0;aba5a916dfea:44109 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;aba5a916dfea:44109. 2024-11-19T12:54:46,721 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T12:54:46,721 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b95c7ead1b6f1bb58c55808f92c9fab, disabling compactions & flushes 2024-11-19T12:54:46,721 DEBUG [RS:0;aba5a916dfea:44109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:46,721 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:46,721 DEBUG [RS:0;aba5a916dfea:44109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:46,721 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:46,722 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T12:54:46,722 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. after waiting 0 ms 2024-11-19T12:54:46,722 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1325): Online Regions={1b95c7ead1b6f1bb58c55808f92c9fab=testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.} 2024-11-19T12:54:46,722 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:46,722 DEBUG [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1351): Waiting on 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:46,728 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T12:54:46,728 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:46,728 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b95c7ead1b6f1bb58c55808f92c9fab: Waiting for close lock at 1732020886721Running coprocessor pre-close hooks at 1732020886721Disabling compacts and flushes for region at 1732020886721Disabling writes for close at 1732020886722 (+1 ms)Writing region close event to WAL at 1732020886723 (+1 ms)Running coprocessor post-close hooks at 1732020886728 (+5 ms)Closed at 1732020886728 2024-11-19T12:54:46,729 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:46,729 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T12:54:46,729 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T12:54:46,730 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:46,730 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:46,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:46,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020886729 2024-11-19T12:54:46,733 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T12:54:46,922 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,44109,1732020883670; all regions closed. 2024-11-19T12:54:46,924 DEBUG [RS:0;aba5a916dfea:44109 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C44109%2C1732020883670:(num 1732020884385) 2024-11-19T12:54:46,924 DEBUG [RS:0;aba5a916dfea:44109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:46,924 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:46,924 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:46,925 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:46,925 INFO [RS:0;aba5a916dfea:44109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:46,925 INFO [RS:0;aba5a916dfea:44109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44109 2024-11-19T12:54:46,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,44109,1732020883670 2024-11-19T12:54:46,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:46,957 INFO [RS:0;aba5a916dfea:44109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:46,968 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,44109,1732020883670] 2024-11-19T12:54:46,976 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,44109,1732020883670 already deleted, retry=false 2024-11-19T12:54:46,977 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of aba5a916dfea,44109,1732020883670 on aba5a916dfea,36439,1732020883517 2024-11-19T12:54:46,986 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure aba5a916dfea,44109,1732020883670, splitWal=true, meta=false 2024-11-19T12:54:46,989 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure aba5a916dfea,44109,1732020883670, splitWal=true, meta=false 2024-11-19T12:54:46,989 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for aba5a916dfea,44109,1732020883670 (carryingMeta=false) aba5a916dfea,44109,1732020883670/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@578d935c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T12:54:46,990 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): aba5a916dfea,44109,1732020883670 had 1 regions 2024-11-19T12:54:46,991 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure aba5a916dfea,44109,1732020883670, splitWal=true, meta=false, isMeta: false 2024-11-19T12:54:46,992 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670-splitting 2024-11-19T12:54:46,992 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670-splitting dir is empty, no logs to split. 2024-11-19T12:54:46,992 INFO [PEWorker-1 {}] master.SplitWALManager(105): aba5a916dfea,44109,1732020883670 WAL count=0, meta=false 2024-11-19T12:54:46,994 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670-splitting dir is empty, no logs to split. 2024-11-19T12:54:46,994 INFO [PEWorker-1 {}] master.SplitWALManager(105): aba5a916dfea,44109,1732020883670 WAL count=0, meta=false 2024-11-19T12:54:46,994 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if aba5a916dfea,44109,1732020883670 WAL splitting is done? wals=0, meta=false 2024-11-19T12:54:46,994 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for aba5a916dfea,44109,1732020883670 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,44109,1732020883670-splitting does not exist 2024-11-19T12:54:46,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN}] 2024-11-19T12:54:46,996 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN 2024-11-19T12:54:46,997 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T12:54:47,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:47,068 INFO [RS:0;aba5a916dfea:44109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:47,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44109-0x1015472e2f20001, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:47,069 INFO [RS:0;aba5a916dfea:44109 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,44109,1732020883670; zookeeper connection closed. 2024-11-19T12:54:47,069 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16da866e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16da866e 2024-11-19T12:54:47,148 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1b95c7ead1b6f1bb58c55808f92c9fab, regionState=OPENING, regionLocation=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:47,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN because future has completed 2024-11-19T12:54:47,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,36557,1732020883709}] 2024-11-19T12:54:47,166 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/aba5a916dfea:0 server-side Connection retries=45 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T12:54:47,166 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T12:54:47,167 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37395 2024-11-19T12:54:47,169 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37395 connecting to ZooKeeper ensemble=127.0.0.1:52121 2024-11-19T12:54:47,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:47,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:47,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373950x0, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T12:54:47,193 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37395-0x1015472e2f20003 connected 2024-11-19T12:54:47,193 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-19T12:54:47,193 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-19T12:54:47,194 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T12:54:47,195 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache 2024-11-19T12:54:47,195 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T12:54:47,195 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T12:54:47,195 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T12:54:47,196 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T12:54:47,196 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T12:54:47,196 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T12:54:47,197 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-19T12:54:47,197 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T12:54:47,197 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-19T12:54:47,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T12:54:47,201 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-19T12:54:47,202 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T12:54:47,202 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-19T12:54:47,202 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence is complete. 2024-11-19T12:54:47,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37395 2024-11-19T12:54:47,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37395 2024-11-19T12:54:47,203 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37395 2024-11-19T12:54:47,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37395 2024-11-19T12:54:47,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37395 2024-11-19T12:54:47,206 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(746): ClusterId : fff98aaa-7ebe-4cd6-a594-2aa139895381 2024-11-19T12:54:47,206 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T12:54:47,216 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T12:54:47,216 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T12:54:47,227 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T12:54:47,228 DEBUG [RS:2;aba5a916dfea:37395 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b75b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=aba5a916dfea/172.17.0.2:0 2024-11-19T12:54:47,238 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;aba5a916dfea:37395 2024-11-19T12:54:47,238 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T12:54:47,238 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T12:54:47,238 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T12:54:47,239 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(2659): reportForDuty to master=aba5a916dfea,36439,1732020883517 with port=37395, startcode=1732020887165 2024-11-19T12:54:47,239 DEBUG [RS:2;aba5a916dfea:37395 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T12:54:47,241 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53615, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T12:54:47,242 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(363): Checking decommissioned status of RegionServer aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,242 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] master.ServerManager(517): Registering regionserver=aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,243 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697 2024-11-19T12:54:47,243 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T12:54:47,243 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T12:54:47,251 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:47,252 DEBUG [RS:2;aba5a916dfea:37395 {}] zookeeper.ZKUtil(111): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,252 WARN [RS:2;aba5a916dfea:37395 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T12:54:47,252 INFO [RS:2;aba5a916dfea:37395 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T12:54:47,252 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [aba5a916dfea,37395,1732020887165] 2024-11-19T12:54:47,252 DEBUG [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,257 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T12:54:47,259 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T12:54:47,259 INFO [RS:2;aba5a916dfea:37395 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T12:54:47,260 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,260 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T12:54:47,260 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T12:54:47,260 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,260 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,260 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/aba5a916dfea:0, corePoolSize=2, maxPoolSize=2 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/aba5a916dfea:0, corePoolSize=1, maxPoolSize=1 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:47,261 DEBUG [RS:2;aba5a916dfea:37395 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/aba5a916dfea:0, corePoolSize=3, maxPoolSize=3 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,262 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,37395,1732020887165-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T12:54:47,277 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T12:54:47,277 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=aba5a916dfea,37395,1732020887165-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,277 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,277 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.Replication(171): aba5a916dfea,37395,1732020887165 started 2024-11-19T12:54:47,292 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T12:54:47,292 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1482): Serving as aba5a916dfea,37395,1732020887165, RpcServer on aba5a916dfea/172.17.0.2:37395, sessionid=0x1015472e2f20003 2024-11-19T12:54:47,292 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T12:54:47,292 DEBUG [RS:2;aba5a916dfea:37395 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,292 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,37395,1732020887165' 2024-11-19T12:54:47,292 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T12:54:47,293 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'aba5a916dfea,37395,1732020887165' 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T12:54:47,294 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T12:54:47,295 DEBUG [RS:2;aba5a916dfea:37395 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T12:54:47,295 INFO [RS:2;aba5a916dfea:37395 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T12:54:47,295 INFO [RS:2;aba5a916dfea:37395 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T12:54:47,306 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,307 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1b95c7ead1b6f1bb58c55808f92c9fab, NAME => 'testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.', STARTKEY => '', ENDKEY => ''} 2024-11-19T12:54:47,307 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732020884959 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,307 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T12:54:47,307 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,307 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,308 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,309 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1b95c7ead1b6f1bb58c55808f92c9fab columnFamilyName family 2024-11-19T12:54:47,309 DEBUG [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T12:54:47,311 DEBUG [StoreFileOpener-1b95c7ead1b6f1bb58c55808f92c9fab-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de, delay=984 ms 2024-11-19T12:54:47,312 DEBUG [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:47,313 INFO [StoreOpener-1b95c7ead1b6f1bb58c55808f92c9fab-1 {}] regionserver.HStore(327): Store=1b95c7ead1b6f1bb58c55808f92c9fab/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T12:54:47,313 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,313 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,314 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,314 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,314 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,315 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,315 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1b95c7ead1b6f1bb58c55808f92c9fab; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63756306, jitterRate=-0.04995700716972351}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T12:54:47,316 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,316 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1b95c7ead1b6f1bb58c55808f92c9fab: Running coprocessor pre-open hook at 1732020887307Writing region info on filesystem at 1732020887307Initializing all the Stores at 1732020887308 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732020887308Cleaning up temporary data from old regions at 1732020887314 (+6 ms)Running coprocessor post-open hooks at 1732020887316 (+2 ms)Region opened successfully at 1732020887316 2024-11-19T12:54:47,317 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab., pid=9, masterSystemTime=1732020887304 2024-11-19T12:54:47,320 DEBUG [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,320 INFO [RS_OPEN_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1b95c7ead1b6f1bb58c55808f92c9fab, regionState=OPEN, openSeqNum=10, regionLocation=aba5a916dfea,36557,1732020883709 2024-11-19T12:54:47,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,36557,1732020883709 because future has completed 2024-11-19T12:54:47,323 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36439 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=aba5a916dfea,36557,1732020883709, table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-19T12:54:47,326 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-19T12:54:47,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1b95c7ead1b6f1bb58c55808f92c9fab, server=aba5a916dfea,36557,1732020883709 in 172 msec 2024-11-19T12:54:47,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T12:54:47,328 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732020884959, region=1b95c7ead1b6f1bb58c55808f92c9fab, ASSIGN in 332 msec 2024-11-19T12:54:47,328 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server aba5a916dfea,44109,1732020883670 after splitting done 2024-11-19T12:54:47,330 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure aba5a916dfea,44109,1732020883670, splitWal=true, meta=false in 350 msec 2024-11-19T12:54:47,396 INFO [RS:2;aba5a916dfea:37395 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T12:54:47,397 INFO [RS:2;aba5a916dfea:37395 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=aba5a916dfea%2C37395%2C1732020887165, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37395,1732020887165, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs, maxLogs=55 2024-11-19T12:54:47,413 INFO [RS:2;aba5a916dfea:37395 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/WALs/aba5a916dfea,37395,1732020887165/aba5a916dfea%2C37395%2C1732020887165.1732020887397 2024-11-19T12:54:47,413 DEBUG [RS:2;aba5a916dfea:37395 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T12:54:47,642 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T12:54:47,643 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T12:54:47,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:54:47,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T12:54:47,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T12:54:47,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T12:54:47,646 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732020884959 2024-11-19T12:54:47,646 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732020884959 Metrics about Tables on a single HBase RegionServer 2024-11-19T12:54:47,792 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T12:54:47,793 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:47,793 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:47,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T12:54:47,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T12:54:47,795 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=719379349, stopped=false 2024-11-19T12:54:47,795 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=aba5a916dfea,36439,1732020883517 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:47,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:47,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:47,857 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T12:54:47,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:47,858 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:47,858 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:47,858 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T12:54:47,858 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,36557,1732020883709' ***** 2024-11-19T12:54:47,858 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:54:47,859 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'aba5a916dfea,37395,1732020887165' ***** 2024-11-19T12:54:47,859 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T12:54:47,859 INFO [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T12:54:47,859 INFO [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T12:54:47,859 INFO [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d159847a-bc8d-c41c-4ce9-09d76ace79a5/bucket.persistence 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T12:54:47,859 INFO [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T12:54:47,860 INFO [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T12:54:47,860 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T12:54:47,860 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T12:54:47,862 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:47,863 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:47,870 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1487): Thread RS:1;aba5a916dfea:36557 started persisting bucket cache to file 2024-11-19T12:54:47,870 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1487): Thread RS:2;aba5a916dfea:37395 started persisting bucket cache to file 2024-11-19T12:54:47,870 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:47,870 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:47,870 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T12:54:47,870 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T12:54:47,872 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:47,873 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020887870 2024-11-19T12:54:47,873 DEBUG [RS:1;aba5a916dfea:36557 {}] bucket.BucketCache(1509): Thread RS:1;aba5a916dfea:36557 finished persisting bucket cache to file, renaming 2024-11-19T12:54:47,873 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:47,873 INFO [RS:1;aba5a916dfea:36557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:47,873 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T12:54:47,873 INFO [RS:1;aba5a916dfea:36557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:47,873 ERROR [RS:2;aba5a916dfea:37395 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T12:54:47,873 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(3091): Received CLOSE for 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,873 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T12:54:47,873 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732020887870 2024-11-19T12:54:47,873 DEBUG [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1509): Thread RS:2;aba5a916dfea:37395 finished persisting bucket cache to file, renaming 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,36557,1732020883709 2024-11-19T12:54:47,874 WARN [RS:2;aba5a916dfea:37395 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;aba5a916dfea:36557. 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T12:54:47,874 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T12:54:47,874 DEBUG [RS:1;aba5a916dfea:36557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:47,874 DEBUG [RS:1;aba5a916dfea:36557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T12:54:47,874 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1b95c7ead1b6f1bb58c55808f92c9fab, disabling compactions & flushes 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(959): stopping server aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,874 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:47,874 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:47,874 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. after waiting 0 ms 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;aba5a916dfea:37395. 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:47,874 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,874 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T12:54:47,874 DEBUG [RS:2;aba5a916dfea:37395 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T12:54:47,874 DEBUG [RS:2;aba5a916dfea:37395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,874 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,37395,1732020887165; all regions closed. 2024-11-19T12:54:47,875 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T12:54:47,875 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1b95c7ead1b6f1bb58c55808f92c9fab=testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab.} 2024-11-19T12:54:47,875 DEBUG [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1b95c7ead1b6f1bb58c55808f92c9fab 2024-11-19T12:54:47,875 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T12:54:47,875 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T12:54:47,875 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T12:54:47,875 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T12:54:47,875 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T12:54:47,875 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T12:54:47,875 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-19T12:54:47,882 DEBUG [StoreFileCloser-1b95c7ead1b6f1bb58c55808f92c9fab-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/family/bb409d63abed430c92dfb6a31fc369de 2024-11-19T12:54:47,883 DEBUG [RS:2;aba5a916dfea:37395 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:47,883 INFO [RS:2;aba5a916dfea:37395 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C37395%2C1732020887165:(num 1732020887397) 2024-11-19T12:54:47,883 DEBUG [RS:2;aba5a916dfea:37395 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:47,883 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:47,883 INFO [RS:2;aba5a916dfea:37395 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T12:54:47,884 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:47,884 INFO [RS:2;aba5a916dfea:37395 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37395 2024-11-19T12:54:47,886 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/default/testBlockEvictionOnGracefulStop1732020884959/1b95c7ead1b6f1bb58c55808f92c9fab/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T12:54:47,887 INFO [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,887 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1b95c7ead1b6f1bb58c55808f92c9fab: Waiting for close lock at 1732020887874Running coprocessor pre-close hooks at 1732020887874Disabling compacts and flushes for region at 1732020887874Disabling writes for close at 1732020887874Writing region close event to WAL at 1732020887882 (+8 ms)Running coprocessor post-close hooks at 1732020887886 (+4 ms)Closed at 1732020887886 2024-11-19T12:54:47,887 DEBUG [RS_CLOSE_REGION-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab. 2024-11-19T12:54:47,889 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 45a411ab88044426a61783112dfdb9ce 2024-11-19T12:54:47,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,37395,1732020887165 2024-11-19T12:54:47,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:47,901 INFO [RS:2;aba5a916dfea:37395 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:47,901 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/45a411ab88044426a61783112dfdb9ce is 209, key is testBlockEvictionOnGracefulStop1732020884959,,1732020884964.1b95c7ead1b6f1bb58c55808f92c9fab./info:regioninfo/1732020887320/Put/seqid=0 2024-11-19T12:54:47,902 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/45a411ab88044426a61783112dfdb9ce 2024-11-19T12:54:47,908 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 31539492488540ec97635c86e4621d91 2024-11-19T12:54:47,909 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,37395,1732020887165] 2024-11-19T12:54:47,917 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,37395,1732020887165 already deleted, retry=false 2024-11-19T12:54:47,917 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aba5a916dfea,37395,1732020887165 expired; onlineServers=1 2024-11-19T12:54:47,920 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/31539492488540ec97635c86e4621d91 is 43, key is default/ns:d/1732020884884/Put/seqid=0 2024-11-19T12:54:47,921 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/31539492488540ec97635c86e4621d91 2024-11-19T12:54:47,927 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8951402835142e5b856693cc1e45468 2024-11-19T12:54:47,940 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/e8951402835142e5b856693cc1e45468 is 80, key is testBlockEvictionOnGracefulStop1732020884959/table:state/1732020885347/Put/seqid=0 2024-11-19T12:54:47,941 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/e8951402835142e5b856693cc1e45468 2024-11-19T12:54:47,943 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/info/45a411ab88044426a61783112dfdb9ce as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/45a411ab88044426a61783112dfdb9ce 2024-11-19T12:54:47,944 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/45a411ab88044426a61783112dfdb9ce, delay=902 ms 2024-11-19T12:54:47,945 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/45a411ab88044426a61783112dfdb9ce, entries=14, sequenceid=13, filesize=7.5 K 2024-11-19T12:54:47,946 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/ns/31539492488540ec97635c86e4621d91 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/31539492488540ec97635c86e4621d91 2024-11-19T12:54:47,947 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/31539492488540ec97635c86e4621d91, delay=984 ms 2024-11-19T12:54:47,948 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/31539492488540ec97635c86e4621d91, entries=2, sequenceid=13, filesize=5.0 K 2024-11-19T12:54:47,948 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/.tmp/table/e8951402835142e5b856693cc1e45468 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/e8951402835142e5b856693cc1e45468 2024-11-19T12:54:47,949 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/e8951402835142e5b856693cc1e45468, delay=928 ms 2024-11-19T12:54:47,950 INFO [regionserver/aba5a916dfea:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:47,950 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/e8951402835142e5b856693cc1e45468, entries=2, sequenceid=13, filesize=5.3 K 2024-11-19T12:54:47,951 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=13, compaction requested=false 2024-11-19T12:54:47,952 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/info/45a411ab88044426a61783112dfdb9ce 2024-11-19T12:54:47,952 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/ns/31539492488540ec97635c86e4621d91 2024-11-19T12:54:47,953 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/table/e8951402835142e5b856693cc1e45468 2024-11-19T12:54:47,957 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-19T12:54:47,957 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T12:54:47,957 INFO [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:47,958 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732020887875Running coprocessor pre-close hooks at 1732020887875Disabling compacts and flushes for region at 1732020887875Disabling writes for close at 1732020887875Obtaining lock to block concurrent updates at 1732020887875Preparing flush snapshotting stores in 1588230740 at 1732020887875Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732020887876 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732020887876Flushing 1588230740/info: creating writer at 1732020887876Flushing 1588230740/info: appending metadata at 1732020887901 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732020887901Flushing 1588230740/ns: creating writer at 1732020887904 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732020887920 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732020887920Flushing 1588230740/table: creating writer at 1732020887922 (+2 ms)Flushing 1588230740/table: appending metadata at 1732020887940 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732020887940Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3495b88d: reopening flushed file at 1732020887943 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@150322bc: reopening flushed file at 1732020887946 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3817286b: reopening flushed file at 1732020887948 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=13, compaction requested=false at 1732020887951 (+3 ms)Writing region close event to WAL at 1732020887953 (+2 ms)Running coprocessor post-close hooks at 1732020887957 (+4 ms)Closed at 1732020887957 2024-11-19T12:54:47,958 DEBUG [RS_CLOSE_META-regionserver/aba5a916dfea:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T12:54:48,010 INFO [RS:2;aba5a916dfea:37395 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:48,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,010 INFO [RS:2;aba5a916dfea:37395 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,37395,1732020887165; zookeeper connection closed. 2024-11-19T12:54:48,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37395-0x1015472e2f20003, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,010 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4034f0b1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4034f0b1 2024-11-19T12:54:48,075 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(976): stopping server aba5a916dfea,36557,1732020883709; all regions closed. 2024-11-19T12:54:48,081 DEBUG [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:48,081 INFO [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C36557%2C1732020883709.meta:.meta(num 1732020884807) 2024-11-19T12:54:48,084 DEBUG [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/oldWALs 2024-11-19T12:54:48,084 INFO [RS:1;aba5a916dfea:36557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL aba5a916dfea%2C36557%2C1732020883709:(num 1732020884384) 2024-11-19T12:54:48,084 DEBUG [RS:1;aba5a916dfea:36557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T12:54:48,084 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T12:54:48,085 INFO [RS:1;aba5a916dfea:36557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:48,085 INFO [RS:1;aba5a916dfea:36557 {}] hbase.ChoreService(370): Chore service for: regionserver/aba5a916dfea:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:48,086 INFO [RS:1;aba5a916dfea:36557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:48,086 INFO [regionserver/aba5a916dfea:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:48,086 INFO [RS:1;aba5a916dfea:36557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36557 2024-11-19T12:54:48,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/aba5a916dfea,36557,1732020883709 2024-11-19T12:54:48,115 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T12:54:48,115 INFO [RS:1;aba5a916dfea:36557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:48,126 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [aba5a916dfea,36557,1732020883709] 2024-11-19T12:54:48,134 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/aba5a916dfea,36557,1732020883709 already deleted, retry=false 2024-11-19T12:54:48,134 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; aba5a916dfea,36557,1732020883709 expired; onlineServers=0 2024-11-19T12:54:48,134 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'aba5a916dfea,36439,1732020883517' ***** 2024-11-19T12:54:48,134 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T12:54:48,134 INFO [M:0;aba5a916dfea:36439 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T12:54:48,135 INFO [M:0;aba5a916dfea:36439 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T12:54:48,135 DEBUG [M:0;aba5a916dfea:36439 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T12:54:48,135 DEBUG [M:0;aba5a916dfea:36439 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T12:54:48,135 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T12:54:48,135 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020884120 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.small.0-1732020884120,5,FailOnTimeoutGroup] 2024-11-19T12:54:48,135 DEBUG [master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020884119 {}] cleaner.HFileCleaner(306): Exit Thread[master/aba5a916dfea:0:becomeActiveMaster-HFileCleaner.large.0-1732020884119,5,FailOnTimeoutGroup] 2024-11-19T12:54:48,135 INFO [M:0;aba5a916dfea:36439 {}] hbase.ChoreService(370): Chore service for: master/aba5a916dfea:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T12:54:48,135 INFO [M:0;aba5a916dfea:36439 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T12:54:48,136 DEBUG [M:0;aba5a916dfea:36439 {}] master.HMaster(1795): Stopping service threads 2024-11-19T12:54:48,136 INFO [M:0;aba5a916dfea:36439 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T12:54:48,136 INFO [M:0;aba5a916dfea:36439 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T12:54:48,136 INFO [M:0;aba5a916dfea:36439 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T12:54:48,136 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T12:54:48,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T12:54:48,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T12:54:48,143 DEBUG [M:0;aba5a916dfea:36439 {}] zookeeper.ZKUtil(347): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T12:54:48,143 WARN [M:0;aba5a916dfea:36439 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T12:54:48,143 INFO [M:0;aba5a916dfea:36439 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/.lastflushedseqids 2024-11-19T12:54:48,151 INFO [M:0;aba5a916dfea:36439 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T12:54:48,151 INFO [M:0;aba5a916dfea:36439 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T12:54:48,151 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T12:54:48,151 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:48,151 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:48,151 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T12:54:48,151 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:48,152 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-19T12:54:48,159 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 21380ea4f39f444aaa191e3028351f8d 2024-11-19T12:54:48,174 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21380ea4f39f444aaa191e3028351f8d is 82, key is hbase:meta,,1/info:regioninfo/1732020884836/Put/seqid=0 2024-11-19T12:54:48,175 INFO [M:0;aba5a916dfea:36439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21380ea4f39f444aaa191e3028351f8d 2024-11-19T12:54:48,181 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 93cfbcb3e3b64b1989b715ae328df654 2024-11-19T12:54:48,195 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93cfbcb3e3b64b1989b715ae328df654 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732020885351/Put/seqid=0 2024-11-19T12:54:48,196 INFO [M:0;aba5a916dfea:36439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93cfbcb3e3b64b1989b715ae328df654 2024-11-19T12:54:48,202 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 01266572b60b4525b4d36d741ee5ff8b 2024-11-19T12:54:48,215 DEBUG [M:0;aba5a916dfea:36439 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01266572b60b4525b4d36d741ee5ff8b is 69, key is aba5a916dfea,36557,1732020883709/rs:state/1732020884182/Put/seqid=0 2024-11-19T12:54:48,216 INFO [M:0;aba5a916dfea:36439 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01266572b60b4525b4d36d741ee5ff8b 2024-11-19T12:54:48,218 INFO [M:0;aba5a916dfea:36439 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01266572b60b4525b4d36d741ee5ff8b 2024-11-19T12:54:48,218 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/21380ea4f39f444aaa191e3028351f8d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21380ea4f39f444aaa191e3028351f8d 2024-11-19T12:54:48,220 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/21380ea4f39f444aaa191e3028351f8d, entries=8, sequenceid=83, filesize=5.5 K 2024-11-19T12:54:48,220 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/93cfbcb3e3b64b1989b715ae328df654 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/93cfbcb3e3b64b1989b715ae328df654 2024-11-19T12:54:48,221 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/93cfbcb3e3b64b1989b715ae328df654, entries=9, sequenceid=83, filesize=6.4 K 2024-11-19T12:54:48,222 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/01266572b60b4525b4d36d741ee5ff8b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01266572b60b4525b4d36d741ee5ff8b 2024-11-19T12:54:48,223 INFO [M:0;aba5a916dfea:36439 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 01266572b60b4525b4d36d741ee5ff8b 2024-11-19T12:54:48,223 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0c81ef30-9f6f-074d-c189-14351de920d5/5f83d8e2-d2d1-b796-438d-67ea51970697/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/01266572b60b4525b4d36d741ee5ff8b, entries=3, sequenceid=83, filesize=5.3 K 2024-11-19T12:54:48,224 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=83, compaction requested=false 2024-11-19T12:54:48,225 INFO [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T12:54:48,226 DEBUG [M:0;aba5a916dfea:36439 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732020888151Disabling compacts and flushes for region at 1732020888151Disabling writes for close at 1732020888151Obtaining lock to block concurrent updates at 1732020888152 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732020888152Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732020888152Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732020888152Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732020888152Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732020888174 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732020888174Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732020888176 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732020888194 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732020888195 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732020888197 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732020888215 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732020888215Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27f5131f: reopening flushed file at 1732020888218 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10dc8285: reopening flushed file at 1732020888220 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53d202eb: reopening flushed file at 1732020888222 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=83, compaction requested=false at 1732020888224 (+2 ms)Writing region close event to WAL at 1732020888225 (+1 ms)Closed at 1732020888225 2024-11-19T12:54:48,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,226 INFO [RS:1;aba5a916dfea:36557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:48,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36557-0x1015472e2f20002, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,226 INFO [RS:1;aba5a916dfea:36557 {}] regionserver.HRegionServer(1031): Exiting; stopping=aba5a916dfea,36557,1732020883709; zookeeper connection closed. 2024-11-19T12:54:48,226 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@66e490e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@66e490e 2024-11-19T12:54:48,226 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-19T12:54:48,227 INFO [M:0;aba5a916dfea:36439 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T12:54:48,227 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T12:54:48,227 INFO [M:0;aba5a916dfea:36439 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36439 2024-11-19T12:54:48,227 INFO [M:0;aba5a916dfea:36439 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T12:54:48,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,351 INFO [M:0;aba5a916dfea:36439 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T12:54:48,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36439-0x1015472e2f20000, quorum=127.0.0.1:52121, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T12:54:48,358 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T12:54:48,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T12:54:48,360 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T12:54:48,367 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=440 (was 478), ProcessCount=11 (was 11), AvailableMemoryMB=6729 (was 6852)