2024-11-08 10:28:31,472 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 10:28:31,492 main DEBUG Took 0.017344 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 10:28:31,492 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 10:28:31,493 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 10:28:31,493 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 10:28:31,495 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,502 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 10:28:31,518 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,522 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,524 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,524 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,525 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,526 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,527 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,527 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,528 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,528 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,529 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,529 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,530 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,531 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,531 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,531 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,532 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,532 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 10:28:31,533 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,533 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 10:28:31,535 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 10:28:31,536 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 10:28:31,537 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 10:28:31,538 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 10:28:31,539 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 10:28:31,540 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 10:28:31,549 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 10:28:31,552 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 10:28:31,555 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 10:28:31,555 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 10:28:31,555 main DEBUG createAppenders(={Console}) 2024-11-08 10:28:31,556 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 10:28:31,557 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 10:28:31,557 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 10:28:31,558 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 10:28:31,558 main DEBUG OutputStream closed 2024-11-08 10:28:31,558 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 10:28:31,559 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 10:28:31,559 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 10:28:31,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 10:28:31,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 10:28:31,640 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 10:28:31,641 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 10:28:31,642 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 10:28:31,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 10:28:31,643 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 10:28:31,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 10:28:31,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 10:28:31,644 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 10:28:31,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 10:28:31,645 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 10:28:31,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 10:28:31,646 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 10:28:31,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 10:28:31,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 10:28:31,647 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 10:28:31,648 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 10:28:31,650 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 10:28:31,651 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 10:28:31,651 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 10:28:31,652 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T10:28:31,663 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-08 10:28:31,666 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 10:28:31,666 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T10:28:31,958 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c 2024-11-08T10:28:31,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=5835 2024-11-08T10:28:32,066 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T10:28:32,154 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:32,168 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/cluster_30810fe6-7bc9-7c24-57aa-887d31522b3d, deleteOnExit=true 2024-11-08T10:28:32,315 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/cluster_30810fe6-7bc9-7c24-57aa-887d31522b3d/zookeeper_0, clientPort=51368, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/cluster_30810fe6-7bc9-7c24-57aa-887d31522b3d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/cluster_30810fe6-7bc9-7c24-57aa-887d31522b3d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T10:28:32,328 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51368 2024-11-08T10:28:32,329 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:32,411 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 with version=8 2024-11-08T10:28:32,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase-staging 2024-11-08T10:28:32,501 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T10:28:32,738 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:32,748 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:32,749 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:32,757 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:32,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:32,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:32,892 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T10:28:32,972 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:28:33,010 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T10:28:33,020 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T10:28:33,024 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:33,061 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 36598 (auto-detected) 2024-11-08T10:28:33,062 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T10:28:33,091 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39745 2024-11-08T10:28:33,117 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39745 connecting to ZooKeeper ensemble=127.0.0.1:51368 2024-11-08T10:28:33,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397450x0, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:33,227 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39745-0x1011b4718660000 connected 2024-11-08T10:28:33,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,301 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,308 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:33,312 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582, hbase.cluster.distributed=false 2024-11-08T10:28:33,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:33,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39745 2024-11-08T10:28:33,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39745 2024-11-08T10:28:33,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39745 2024-11-08T10:28:33,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39745 2024-11-08T10:28:33,342 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39745 2024-11-08T10:28:33,428 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:33,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,431 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:33,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:33,434 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:28:33,438 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:33,439 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34437 2024-11-08T10:28:33,442 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34437 connecting to ZooKeeper ensemble=127.0.0.1:51368 2024-11-08T10:28:33,442 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,443 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,464 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344370x0, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:33,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344370x0, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:33,465 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34437-0x1011b4718660001 connected 2024-11-08T10:28:33,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:28:33,484 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.cache 2024-11-08T10:28:33,490 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T10:28:33,492 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T10:28:33,494 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T10:28:33,495 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T10:28:33,496 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T10:28:33,501 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T10:28:33,506 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:28:33,507 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T10:28:33,508 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence is complete. 2024-11-08T10:28:33,509 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:28:33,514 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:33,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-08T10:28:33,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34437 2024-11-08T10:28:33,517 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34437 2024-11-08T10:28:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-08T10:28:33,518 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34437 2024-11-08T10:28:33,537 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:33,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,538 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:33,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:33,539 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:33,539 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:28:33,539 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:33,540 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39659 2024-11-08T10:28:33,543 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39659 connecting to ZooKeeper ensemble=127.0.0.1:51368 2024-11-08T10:28:33,543 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,544 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396590x0, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:33,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396590x0, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:33,571 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39659-0x1011b4718660002 connected 2024-11-08T10:28:33,572 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:28:33,575 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.cache 2024-11-08T10:28:33,575 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T10:28:33,575 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T10:28:33,576 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T10:28:33,576 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T10:28:33,576 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T10:28:33,577 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T10:28:33,577 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T10:28:33,577 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence is complete. 2024-11-08T10:28:33,578 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:28:33,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:28:33,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:33,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39659 2024-11-08T10:28:33,587 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39659 2024-11-08T10:28:33,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39659 2024-11-08T10:28:33,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39659 2024-11-08T10:28:33,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39659 2024-11-08T10:28:33,619 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0e58e25f69ec:39745 2024-11-08T10:28:33,623 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:33,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,640 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,645 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:33,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:33,687 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:33,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,687 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,689 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T10:28:33,691 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0e58e25f69ec,39745,1731061712578 from backup master directory 2024-11-08T10:28:33,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:33,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,703 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:33,705 WARN [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:33,705 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:33,708 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T10:28:33,710 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T10:28:33,722 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase.id] with ID: eaa884fd-9e8b-4bd9-80f4-8f663de72ac2 2024-11-08T10:28:33,723 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.tmp/hbase.id 2024-11-08T10:28:33,729 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase.id] 2024-11-08T10:28:33,767 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:33,774 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T10:28:33,802 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-08T10:28:33,837 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,837 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:33,863 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:28:33,868 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T10:28:33,908 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:28:33,913 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:33,969 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store 2024-11-08T10:28:33,989 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T10:28:33,992 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:33,994 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:28:33,994 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:33,994 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:33,996 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:28:33,996 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:33,996 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:33,997 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731061713994Disabling compacts and flushes for region at 1731061713994Disabling writes for close at 1731061713996 (+2 ms)Writing region close event to WAL at 1731061713996Closed at 1731061713996 2024-11-08T10:28:33,997 WARN [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/.initializing 2024-11-08T10:28:33,998 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:34,008 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:34,020 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0e58e25f69ec%2C39745%2C1731061712578, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,39745,1731061712578, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/oldWALs, maxLogs=10 2024-11-08T10:28:34,055 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,39745,1731061712578/0e58e25f69ec%2C39745%2C1731061712578.1731061714025 2024-11-08T10:28:34,056 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:34,057 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:34,057 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:34,060 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,062 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T10:28:34,145 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,147 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:34,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T10:28:34,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:34,154 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T10:28:34,160 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:34,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T10:28:34,166 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:34,167 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,172 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,174 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,178 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,179 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,182 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:34,185 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:34,193 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:34,194 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73009234, jitterRate=0.0879223644733429}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:34,204 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731061714074Initializing all the Stores at 1731061714074Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061714075 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061714075Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061714076 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061714076Cleaning up temporary data from old regions at 1731061714179 (+103 ms)Region opened successfully at 1731061714204 (+25 ms) 2024-11-08T10:28:34,206 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T10:28:34,247 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36085e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:34,287 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T10:28:34,306 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T10:28:34,307 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T10:28:34,312 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T10:28:34,313 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T10:28:34,322 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-08T10:28:34,322 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T10:28:34,357 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T10:28:34,371 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T10:28:34,439 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T10:28:34,443 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T10:28:34,445 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T10:28:34,462 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T10:28:34,465 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T10:28:34,472 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T10:28:34,480 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T10:28:34,483 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T10:28:34,495 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T10:28:34,519 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T10:28:34,530 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T10:28:34,555 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:34,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:34,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:34,556 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,559 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0e58e25f69ec,39745,1731061712578, sessionid=0x1011b4718660000, setting cluster-up flag (Was=false) 2024-11-08T10:28:34,589 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,614 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T10:28:34,616 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:34,639 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:34,664 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T10:28:34,666 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:34,671 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T10:28:34,707 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(746): ClusterId : eaa884fd-9e8b-4bd9-80f4-8f663de72ac2 2024-11-08T10:28:34,707 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(746): ClusterId : eaa884fd-9e8b-4bd9-80f4-8f663de72ac2 2024-11-08T10:28:34,709 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:28:34,709 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:28:34,724 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:28:34,724 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:28:34,724 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:28:34,724 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:28:34,732 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:28:34,732 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:28:34,732 DEBUG [RS:1;0e58e25f69ec:39659 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6915ecd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:34,732 DEBUG [RS:0;0e58e25f69ec:34437 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27e7b88f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:34,747 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0e58e25f69ec:39659 2024-11-08T10:28:34,747 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0e58e25f69ec:34437 2024-11-08T10:28:34,751 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T10:28:34,751 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T10:28:34,751 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T10:28:34,751 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T10:28:34,751 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T10:28:34,751 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T10:28:34,750 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:34,755 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(2659): reportForDuty to master=0e58e25f69ec,39745,1731061712578 with port=39659, startcode=1731061713537 2024-11-08T10:28:34,755 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(2659): reportForDuty to master=0e58e25f69ec,39745,1731061712578 with port=34437, startcode=1731061713391 2024-11-08T10:28:34,762 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T10:28:34,769 DEBUG [RS:1;0e58e25f69ec:39659 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:28:34,769 DEBUG [RS:0;0e58e25f69ec:34437 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:28:34,770 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T10:28:34,776 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0e58e25f69ec,39745,1731061712578 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T10:28:34,783 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:34,783 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:34,784 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:34,784 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:34,784 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0e58e25f69ec:0, corePoolSize=10, maxPoolSize=10 2024-11-08T10:28:34,784 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:34,785 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:34,785 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:34,790 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:34,790 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731061744790 2024-11-08T10:28:34,790 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T10:28:34,792 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T10:28:34,793 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T10:28:34,796 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T10:28:34,796 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,796 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T10:28:34,798 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T10:28:34,798 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T10:28:34,796 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T10:28:34,810 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:34,821 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T10:28:34,822 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T10:28:34,823 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T10:28:34,825 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T10:28:34,826 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T10:28:34,839 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061714827,5,FailOnTimeoutGroup] 2024-11-08T10:28:34,841 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061714839,5,FailOnTimeoutGroup] 2024-11-08T10:28:34,841 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:34,841 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T10:28:34,843 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:34,843 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:34,852 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T10:28:34,852 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:34,858 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39409, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:28:34,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44279, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:28:34,865 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39745 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:34,869 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39745 {}] master.ServerManager(517): Registering regionserver=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:34,872 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:34,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T10:28:34,880 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39745 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:34,881 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39745 {}] master.ServerManager(517): Registering regionserver=0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:34,881 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:28:34,881 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,881 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:34,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T10:28:34,885 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:34,885 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T10:28:34,886 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T10:28:34,886 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:34,886 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T10:28:34,886 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T10:28:34,887 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T10:28:34,887 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:34,888 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T10:28:34,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:28:34,893 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,893 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:34,894 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T10:28:34,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:28:34,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:34,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:34,899 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T10:28:34,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:34,900 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T10:28:34,901 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T10:28:34,901 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:34,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T10:28:34,908 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:34,908 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60785431, jitterRate=-0.09422649443149567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:34,914 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731061714872Initializing all the Stores at 1731061714873 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061714873Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061714876 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061714876Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061714876Cleaning up temporary data from old regions at 1731061714901 (+25 ms)Region opened successfully at 1731061714914 (+13 ms) 2024-11-08T10:28:34,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:28:34,915 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T10:28:34,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T10:28:34,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:28:34,915 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:28:34,917 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:34,917 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731061714914Disabling compacts and flushes for region at 1731061714914Disabling writes for close at 1731061714915 (+1 ms)Writing region close event to WAL at 1731061714916 (+1 ms)Closed at 1731061714917 (+1 ms) 2024-11-08T10:28:34,919 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:34,919 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T10:28:34,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T10:28:34,939 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:34,942 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T10:28:34,958 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:34,959 DEBUG [RS:0;0e58e25f69ec:34437 {}] zookeeper.ZKUtil(111): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:34,959 DEBUG [RS:1;0e58e25f69ec:39659 {}] zookeeper.ZKUtil(111): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:34,959 WARN [RS:1;0e58e25f69ec:39659 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:34,959 WARN [RS:0;0e58e25f69ec:34437 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:34,959 INFO [RS:0;0e58e25f69ec:34437 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:34,959 INFO [RS:1;0e58e25f69ec:39659 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:34,959 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:34,959 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:34,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0e58e25f69ec,34437,1731061713391] 2024-11-08T10:28:34,962 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0e58e25f69ec,39659,1731061713537] 2024-11-08T10:28:34,993 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:28:34,993 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:28:35,010 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:28:35,010 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:28:35,015 INFO [RS:1;0e58e25f69ec:39659 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:28:35,015 INFO [RS:0;0e58e25f69ec:34437 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:28:35,015 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,015 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,017 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T10:28:35,017 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T10:28:35,024 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T10:28:35,024 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T10:28:35,026 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,026 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,027 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,027 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:35,028 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:35,028 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,028 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,029 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,029 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,029 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,029 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:35,029 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:35,029 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:35,029 DEBUG [RS:1;0e58e25f69ec:39659 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:35,029 DEBUG [RS:0;0e58e25f69ec:34437 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:35,035 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,035 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,035 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,035 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,34437,1731061713391-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:35,036 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,037 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,037 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39659,1731061713537-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:35,059 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:28:35,061 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,34437,1731061713391-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,062 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,062 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.Replication(171): 0e58e25f69ec,34437,1731061713391 started 2024-11-08T10:28:35,063 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:28:35,064 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39659,1731061713537-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,064 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,064 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.Replication(171): 0e58e25f69ec,39659,1731061713537 started 2024-11-08T10:28:35,082 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,082 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1482): Serving as 0e58e25f69ec,34437,1731061713391, RpcServer on 0e58e25f69ec/172.17.0.2:34437, sessionid=0x1011b4718660001 2024-11-08T10:28:35,083 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:28:35,083 DEBUG [RS:0;0e58e25f69ec:34437 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:35,084 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,34437,1731061713391' 2024-11-08T10:28:35,084 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:28:35,085 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:28:35,085 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:28:35,086 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:28:35,086 DEBUG [RS:0;0e58e25f69ec:34437 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:35,086 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,34437,1731061713391' 2024-11-08T10:28:35,086 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:28:35,087 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:28:35,088 DEBUG [RS:0;0e58e25f69ec:34437 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:28:35,088 INFO [RS:0;0e58e25f69ec:34437 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:28:35,088 INFO [RS:0;0e58e25f69ec:34437 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:28:35,088 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,089 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1482): Serving as 0e58e25f69ec,39659,1731061713537, RpcServer on 0e58e25f69ec/172.17.0.2:39659, sessionid=0x1011b4718660002 2024-11-08T10:28:35,089 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:28:35,089 DEBUG [RS:1;0e58e25f69ec:39659 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:35,089 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,39659,1731061713537' 2024-11-08T10:28:35,089 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:28:35,090 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:28:35,091 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:28:35,091 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:28:35,091 DEBUG [RS:1;0e58e25f69ec:39659 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:35,092 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,39659,1731061713537' 2024-11-08T10:28:35,092 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:28:35,093 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:28:35,093 WARN [0e58e25f69ec:39745 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:28:35,093 DEBUG [RS:1;0e58e25f69ec:39659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:28:35,093 INFO [RS:1;0e58e25f69ec:39659 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:28:35,093 INFO [RS:1;0e58e25f69ec:39659 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:28:35,193 INFO [RS:0;0e58e25f69ec:34437 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:35,195 INFO [RS:1;0e58e25f69ec:39659 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:35,195 INFO [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C34437%2C1731061713391, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,34437,1731061713391, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:35,196 INFO [RS:1;0e58e25f69ec:39659 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C39659%2C1731061713537, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,39659,1731061713537, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:35,217 INFO [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,34437,1731061713391/0e58e25f69ec%2C34437%2C1731061713391.1731061715197 2024-11-08T10:28:35,218 DEBUG [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:35,224 INFO [RS:1;0e58e25f69ec:39659 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,39659,1731061713537/0e58e25f69ec%2C39659%2C1731061713537.1731061715199 2024-11-08T10:28:35,225 DEBUG [RS:1;0e58e25f69ec:39659 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:35,346 DEBUG [0e58e25f69ec:39745 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T10:28:35,360 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(204): Hosts are {0e58e25f69ec=0} racks are {/default-rack=0} 2024-11-08T10:28:35,365 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T10:28:35,365 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T10:28:35,365 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T10:28:35,365 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T10:28:35,365 INFO [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T10:28:35,365 INFO [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T10:28:35,365 DEBUG [0e58e25f69ec:39745 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T10:28:35,373 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:35,378 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,34437,1731061713391, state=OPENING 2024-11-08T10:28:35,413 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T10:28:35,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:35,422 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:35,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:35,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,423 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,425 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:35,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,34437,1731061713391}] 2024-11-08T10:28:35,597 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T10:28:35,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T10:28:35,611 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T10:28:35,611 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:35,612 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T10:28:35,614 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C34437%2C1731061713391.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,34437,1731061713391, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:35,639 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,34437,1731061713391/0e58e25f69ec%2C34437%2C1731061713391.meta.1731061715617.meta 2024-11-08T10:28:35,640 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:35,640 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:35,642 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T10:28:35,645 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T10:28:35,651 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T10:28:35,655 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T10:28:35,656 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:35,656 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T10:28:35,656 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T10:28:35,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T10:28:35,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:28:35,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:35,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:35,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T10:28:35,661 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T10:28:35,661 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:35,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:35,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T10:28:35,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:28:35,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:35,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:35,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T10:28:35,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:28:35,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:35,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:35,665 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T10:28:35,667 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:35,667 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:35,668 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T10:28:35,668 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T10:28:35,668 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:35,669 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T10:28:35,670 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72561991, jitterRate=0.08125792443752289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:35,670 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T10:28:35,672 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731061715657Writing region info on filesystem at 1731061715657Initializing all the Stores at 1731061715657Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061715657Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061715658 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061715658Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061715658Cleaning up temporary data from old regions at 1731061715668 (+10 ms)Running coprocessor post-open hooks at 1731061715670 (+2 ms)Region opened successfully at 1731061715672 (+2 ms) 2024-11-08T10:28:35,678 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731061715589 2024-11-08T10:28:35,689 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T10:28:35,689 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T10:28:35,690 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:35,694 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,34437,1731061713391, state=OPEN 2024-11-08T10:28:35,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:35,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:35,722 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:35,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,722 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:35,722 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:35,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T10:28:35,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,34437,1731061713391 in 297 msec 2024-11-08T10:28:35,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T10:28:35,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 799 msec 2024-11-08T10:28:35,731 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:35,731 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T10:28:35,749 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T10:28:35,750 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,34437,1731061713391, seqNum=-1] 2024-11-08T10:28:35,771 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:28:35,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45841, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:28:35,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0920 sec 2024-11-08T10:28:35,792 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731061715792, completionTime=-1 2024-11-08T10:28:35,795 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T10:28:35,795 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T10:28:35,828 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T10:28:35,828 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731061775828 2024-11-08T10:28:35,828 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731061835828 2024-11-08T10:28:35,828 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-08T10:28:35,830 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T10:28:35,837 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,838 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,838 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,839 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0e58e25f69ec:39745, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,839 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,840 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,845 DEBUG [master/0e58e25f69ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T10:28:35,871 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.166sec 2024-11-08T10:28:35,873 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T10:28:35,874 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T10:28:35,875 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T10:28:35,876 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T10:28:35,876 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T10:28:35,877 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:35,877 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T10:28:35,879 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T10:28:35,880 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T10:28:35,880 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,39745,1731061712578-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:35,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8eba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:35,921 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T10:28:35,921 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T10:28:35,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0e58e25f69ec,39745,-1 for getting cluster id 2024-11-08T10:28:35,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T10:28:35,936 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'eaa884fd-9e8b-4bd9-80f4-8f663de72ac2' 2024-11-08T10:28:35,938 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T10:28:35,939 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "eaa884fd-9e8b-4bd9-80f4-8f663de72ac2" 2024-11-08T10:28:35,941 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149feca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:35,941 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0e58e25f69ec,39745,-1] 2024-11-08T10:28:35,944 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T10:28:35,946 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:35,948 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T10:28:35,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201cd73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:35,952 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T10:28:35,961 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,34437,1731061713391, seqNum=-1] 2024-11-08T10:28:35,962 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:28:35,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:28:35,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:35,989 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T10:28:35,994 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:35,996 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@50aedccb 2024-11-08T10:28:35,997 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T10:28:35,999 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T10:28:36,015 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731061715986', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:28:36,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 2024-11-08T10:28:36,025 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:28:36,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731061715986" procId is: 4 2024-11-08T10:28:36,028 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:36,032 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:28:36,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:36,045 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 48e478bc9e54df81ffb6bf52e5f85fdf, NAME => 'testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731061715986', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:36,051 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:36,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1722): Closing 48e478bc9e54df81ffb6bf52e5f85fdf, disabling compactions & flushes 2024-11-08T10:28:36,052 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. after waiting 0 ms 2024-11-08T10:28:36,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,052 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,052 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731061715986-pool-0 {}] regionserver.HRegion(1676): Region close journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Waiting for close lock at 1731061716051Disabling compacts and flushes for region at 1731061716051Disabling writes for close at 1731061716052 (+1 ms)Writing region close event to WAL at 1731061716052Closed at 1731061716052 2024-11-08T10:28:36,054 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:28:36,059 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731061716054"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061716054"}]},"ts":"1731061716054"} 2024-11-08T10:28:36,068 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T10:28:36,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:28:36,073 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731061715986","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061716070"}]},"ts":"1731061716070"} 2024-11-08T10:28:36,078 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731061715986, state=ENABLING in hbase:meta 2024-11-08T10:28:36,079 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0e58e25f69ec=0} racks are {/default-rack=0} 2024-11-08T10:28:36,079 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T10:28:36,079 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T10:28:36,080 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T10:28:36,080 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T10:28:36,080 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T10:28:36,080 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T10:28:36,080 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T10:28:36,081 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, ASSIGN}] 2024-11-08T10:28:36,083 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, ASSIGN 2024-11-08T10:28:36,085 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, ASSIGN; state=OFFLINE, location=0e58e25f69ec,34437,1731061713391; forceNewPlan=false, retain=false 2024-11-08T10:28:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:36,237 INFO [0e58e25f69ec:39745 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T10:28:36,238 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=OPENING, regionLocation=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:36,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, ASSIGN because future has completed 2024-11-08T10:28:36,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391}] 2024-11-08T10:28:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:36,402 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,402 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 48e478bc9e54df81ffb6bf52e5f85fdf, NAME => 'testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:36,403 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731061715986 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,403 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:36,403 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,403 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,423 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,427 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e478bc9e54df81ffb6bf52e5f85fdf columnFamilyName family 2024-11-08T10:28:36,427 DEBUG [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:36,428 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(327): Store=48e478bc9e54df81ffb6bf52e5f85fdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:36,428 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,429 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,429 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,430 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,430 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,431 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,436 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:36,437 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 48e478bc9e54df81ffb6bf52e5f85fdf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74155706, jitterRate=0.1050061285495758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:28:36,437 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:36,438 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Running coprocessor pre-open hook at 1731061716404Writing region info on filesystem at 1731061716404Initializing all the Stores at 1731061716404Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061716404Cleaning up temporary data from old regions at 1731061716430 (+26 ms)Running coprocessor post-open hooks at 1731061716437 (+7 ms)Region opened successfully at 1731061716438 (+1 ms) 2024-11-08T10:28:36,440 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf., pid=6, masterSystemTime=1731061716398 2024-11-08T10:28:36,443 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,443 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:36,443 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=OPEN, openSeqNum=2, regionLocation=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:36,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391 because future has completed 2024-11-08T10:28:36,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T10:28:36,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391 in 206 msec 2024-11-08T10:28:36,452 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T10:28:36,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, ASSIGN in 370 msec 2024-11-08T10:28:36,454 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:28:36,454 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731061715986","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061716454"}]},"ts":"1731061716454"} 2024-11-08T10:28:36,457 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731061715986, state=ENABLED in hbase:meta 2024-11-08T10:28:36,459 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:28:36,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731061715986 in 440 msec 2024-11-08T10:28:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:36,669 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731061715986 get assigned. Timeout = 60000ms 2024-11-08T10:28:36,669 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731061715986 completed 2024-11-08T10:28:36,672 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:28:36,678 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731061715986 assigned to meta. Checking AM states. 2024-11-08T10:28:36,679 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:28:36,679 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731061715986 assigned. 2024-11-08T10:28:36,689 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731061715986', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf., hostname=0e58e25f69ec,34437,1731061713391, seqNum=2] 2024-11-08T10:28:36,701 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 48e478bc9e54df81ffb6bf52e5f85fdf 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T10:28:36,727 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:36,775 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/.tmp/family/b593a2d35e824f9ca8ed1e61728b3f6a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T10:28:36,781 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/.tmp/family/b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:36,826 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/.tmp/family/b593a2d35e824f9ca8ed1e61728b3f6a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:36,827 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, delay=909 ms 2024-11-08T10:28:36,830 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T10:28:36,838 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 48e478bc9e54df81ffb6bf52e5f85fdf in 136ms, sequenceid=6, compaction requested=false 2024-11-08T10:28:36,839 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731061715986' 2024-11-08T10:28:36,841 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 48e478bc9e54df81ffb6bf52e5f85fdf: 2024-11-08T10:28:37,739 DEBUG [hfile-prefetch-1731061716828 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b593a2d35e824f9ca8ed1e61728b3f6a_0. 2024-11-08T10:28:37,742 DEBUG [hfile-prefetch-1731061716828 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b593a2d35e824f9ca8ed1e61728b3f6a_117. 2024-11-08T10:28:37,743 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, with total blocks 2, and data blocks 1 2024-11-08T10:28:37,743 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T10:28:37,744 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T10:28:37,744 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a. Will sleep 100ms and try the verification again. 2024-11-08T10:28:37,844 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, with total blocks 2, and data blocks 1 2024-11-08T10:28:37,844 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-08T10:28:37,845 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2294): found block for file b593a2d35e824f9ca8ed1e61728b3f6a in the backing map. Acquiring read lock for offset 0 2024-11-08T10:28:37,845 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a has now been fully cached. 2024-11-08T10:28:37,845 DEBUG [hfile-prefetch-1731061716828 {}] hfile.PrefetchExecutor(121): Prefetch completed for b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:37,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=48e478bc9e54df81ffb6bf52e5f85fdf, source=0e58e25f69ec,34437,1731061713391, destination=0e58e25f69ec,39659,1731061713537, warming up region on 0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T10:28:37,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=48e478bc9e54df81ffb6bf52e5f85fdf, source=0e58e25f69ec,34437,1731061713391, destination=0e58e25f69ec,39659,1731061713537, running balancer 2024-11-08T10:28:37,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE 2024-11-08T10:28:37,857 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34027, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T10:28:37,857 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE 2024-11-08T10:28:37,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(7855): Warmup {ENCODED => 48e478bc9e54df81ffb6bf52e5f85fdf, NAME => 'testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:37,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=CLOSING, regionLocation=0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:37,863 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:37,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE because future has completed 2024-11-08T10:28:37,863 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-08T10:28:37,863 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e478bc9e54df81ffb6bf52e5f85fdf columnFamilyName family 2024-11-08T10:28:37,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391}] 2024-11-08T10:28:37,863 DEBUG [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:37,867 DEBUG [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:37,867 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(327): Store=48e478bc9e54df81ffb6bf52e5f85fdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1722): Closing 48e478bc9e54df81ffb6bf52e5f85fdf, disabling compactions & flushes 2024-11-08T10:28:37,868 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. after waiting 0 ms 2024-11-08T10:28:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:37,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39659 {}] regionserver.HRegion(1676): Region close journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Waiting for close lock at 1731061717868Disabling compacts and flushes for region at 1731061717868Disabling writes for close at 1731061717868Writing region close event to WAL at 1731061717870 (+2 ms)Closed at 1731061717870 2024-11-08T10:28:37,879 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T10:28:37,879 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T10:28:37,880 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:37,880 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:37,894 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:37,894 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061717879 2024-11-08T10:28:37,895 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T10:28:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39745 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-08T10:28:38,021 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,022 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-08T10:28:38,022 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 48e478bc9e54df81ffb6bf52e5f85fdf, disabling compactions & flushes 2024-11-08T10:28:38,022 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,022 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,022 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. after waiting 0 ms 2024-11-08T10:28:38,023 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,024 DEBUG [StoreFileCloser-48e478bc9e54df81ffb6bf52e5f85fdf-family-1 {}] bucket.BucketCache(769): Removing file b593a2d35e824f9ca8ed1e61728b3f6a for region 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,024 DEBUG [StoreFileCloser-48e478bc9e54df81ffb6bf52e5f85fdf-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file b593a2d35e824f9ca8ed1e61728b3f6a, starting offset: 0, end offset: 9223372036854775807 2024-11-08T10:28:38,025 DEBUG [StoreFileCloser-48e478bc9e54df81ffb6bf52e5f85fdf-family-1 {}] bucket.BucketCache(857): removed key b593a2d35e824f9ca8ed1e61728b3f6a_0 from back map with offset lock 2094080 in the evict process 2024-11-08T10:28:38,030 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T10:28:38,032 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,032 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Waiting for close lock at 1731061718022Running coprocessor pre-close hooks at 1731061718022Disabling compacts and flushes for region at 1731061718022Disabling writes for close at 1731061718022Writing region close event to WAL at 1731061718026 (+4 ms)Running coprocessor post-close hooks at 1731061718031 (+5 ms)Closed at 1731061718032 (+1 ms) 2024-11-08T10:28:38,032 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 48e478bc9e54df81ffb6bf52e5f85fdf move to 0e58e25f69ec,39659,1731061713537 record at close sequenceid=6 2024-11-08T10:28:38,036 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,036 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=CLOSED 2024-11-08T10:28:38,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391 because future has completed 2024-11-08T10:28:38,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T10:28:38,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,34437,1731061713391 in 178 msec 2024-11-08T10:28:38,044 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE; state=CLOSED, location=0e58e25f69ec,39659,1731061713537; forceNewPlan=false, retain=false 2024-11-08T10:28:38,095 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T10:28:38,095 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T10:28:38,095 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:38,095 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061718095 2024-11-08T10:28:38,098 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T10:28:38,103 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:38,103 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061718098 2024-11-08T10:28:38,103 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T10:28:38,195 INFO [0e58e25f69ec:39745 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T10:28:38,195 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=OPENING, regionLocation=0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:38,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE because future has completed 2024-11-08T10:28:38,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,39659,1731061713537}] 2024-11-08T10:28:38,356 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,356 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 48e478bc9e54df81ffb6bf52e5f85fdf, NAME => 'testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:38,357 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731061715986 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,357 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:38,357 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,358 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,359 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,361 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 48e478bc9e54df81ffb6bf52e5f85fdf columnFamilyName family 2024-11-08T10:28:38,361 DEBUG [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:38,364 DEBUG [StoreFileOpener-48e478bc9e54df81ffb6bf52e5f85fdf-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a, delay=989 ms 2024-11-08T10:28:38,366 DEBUG [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:38,366 INFO [StoreOpener-48e478bc9e54df81ffb6bf52e5f85fdf-1 {}] regionserver.HStore(327): Store=48e478bc9e54df81ffb6bf52e5f85fdf/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:38,367 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,368 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,369 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,369 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,370 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,376 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,377 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 48e478bc9e54df81ffb6bf52e5f85fdf; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64886568, jitterRate=-0.03311479091644287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:28:38,378 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,379 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Running coprocessor pre-open hook at 1731061718358Writing region info on filesystem at 1731061718358Initializing all the Stores at 1731061718359 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061718359Cleaning up temporary data from old regions at 1731061718370 (+11 ms)Running coprocessor post-open hooks at 1731061718378 (+8 ms)Region opened successfully at 1731061718379 (+1 ms) 2024-11-08T10:28:38,380 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf., pid=9, masterSystemTime=1731061718353 2024-11-08T10:28:38,385 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,385 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=48e478bc9e54df81ffb6bf52e5f85fdf, regionState=OPEN, openSeqNum=10, regionLocation=0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:38,385 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,39659,1731061713537 because future has completed 2024-11-08T10:28:38,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-08T10:28:38,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 48e478bc9e54df81ffb6bf52e5f85fdf, server=0e58e25f69ec,39659,1731061713537 in 195 msec 2024-11-08T10:28:38,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731061715986, region=48e478bc9e54df81ffb6bf52e5f85fdf, REOPEN/MOVE in 542 msec 2024-11-08T10:28:38,410 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T10:28:38,410 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T10:28:38,411 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:38,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T10:28:38,416 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T10:28:38,416 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=520236456, stopped=false 2024-11-08T10:28:38,417 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0e58e25f69ec,39745,1731061712578 2024-11-08T10:28:38,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:38,447 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:38,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:38,447 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:38,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:38,448 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T10:28:38,449 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T10:28:38,449 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:38,450 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,450 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0e58e25f69ec,34437,1731061713391' ***** 2024-11-08T10:28:38,450 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T10:28:38,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:38,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:38,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:38,451 INFO [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T10:28:38,451 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0e58e25f69ec,39659,1731061713537' ***** 2024-11-08T10:28:38,451 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T10:28:38,451 INFO [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence 2024-11-08T10:28:38,452 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T10:28:38,452 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:38,452 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T10:28:38,452 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T10:28:38,453 INFO [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T10:28:38,453 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T10:28:38,453 INFO [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T10:28:38,454 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1488): Thread RS:0;0e58e25f69ec:34437 started persisting bucket cache to file 2024-11-08T10:28:38,454 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:38,454 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:38,454 INFO [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/bucket.persistence 2024-11-08T10:28:38,454 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T10:28:38,454 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T10:28:38,455 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T10:28:38,455 INFO [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T10:28:38,455 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T10:28:38,458 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:38,458 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061718454 2024-11-08T10:28:38,458 DEBUG [RS:0;0e58e25f69ec:34437 {}] bucket.BucketCache(1510): Thread RS:0;0e58e25f69ec:34437 finished persisting bucket cache to file, renaming 2024-11-08T10:28:38,459 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:28:38,459 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T10:28:38,459 INFO [RS:0;0e58e25f69ec:34437 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:28:38,459 INFO [RS:0;0e58e25f69ec:34437 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:28:38,459 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(959): stopping server 0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0e58e25f69ec:34437. 2024-11-08T10:28:38,460 DEBUG [RS:0;0e58e25f69ec:34437 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:38,460 DEBUG [RS:0;0e58e25f69ec:34437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:28:38,460 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T10:28:38,461 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T10:28:38,461 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T10:28:38,462 DEBUG [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T10:28:38,462 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:38,462 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:28:38,462 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T10:28:38,462 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T10:28:38,462 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:28:38,463 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:28:38,463 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-08T10:28:38,465 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1488): Thread RS:1;0e58e25f69ec:39659 started persisting bucket cache to file 2024-11-08T10:28:38,465 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:38,465 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:38,485 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:38,485 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061718465 2024-11-08T10:28:38,485 DEBUG [RS:1;0e58e25f69ec:39659 {}] bucket.BucketCache(1510): Thread RS:1;0e58e25f69ec:39659 finished persisting bucket cache to file, renaming 2024-11-08T10:28:38,486 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:28:38,486 INFO [RS:1;0e58e25f69ec:39659 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:28:38,486 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T10:28:38,486 INFO [RS:1;0e58e25f69ec:39659 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:28:38,486 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(3091): Received CLOSE for 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,487 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(959): stopping server 0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:38,487 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:38,487 INFO [RS:1;0e58e25f69ec:39659 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0e58e25f69ec:39659. 2024-11-08T10:28:38,487 DEBUG [RS:1;0e58e25f69ec:39659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:38,487 DEBUG [RS:1;0e58e25f69ec:39659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,487 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T10:28:38,487 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1325): Online Regions={48e478bc9e54df81ffb6bf52e5f85fdf=testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf.} 2024-11-08T10:28:38,487 DEBUG [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1351): Waiting on 48e478bc9e54df81ffb6bf52e5f85fdf 2024-11-08T10:28:38,488 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:38,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 48e478bc9e54df81ffb6bf52e5f85fdf, disabling compactions & flushes 2024-11-08T10:28:38,488 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. after waiting 0 ms 2024-11-08T10:28:38,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,491 DEBUG [StoreFileCloser-48e478bc9e54df81ffb6bf52e5f85fdf-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/family/b593a2d35e824f9ca8ed1e61728b3f6a 2024-11-08T10:28:38,496 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnRegionMove1731061715986/48e478bc9e54df81ffb6bf52e5f85fdf/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T10:28:38,497 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,497 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 48e478bc9e54df81ffb6bf52e5f85fdf: Waiting for close lock at 1731061718488Running coprocessor pre-close hooks at 1731061718488Disabling compacts and flushes for region at 1731061718488Disabling writes for close at 1731061718488Writing region close event to WAL at 1731061718491 (+3 ms)Running coprocessor post-close hooks at 1731061718497 (+6 ms)Closed at 1731061718497 2024-11-08T10:28:38,497 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf. 2024-11-08T10:28:38,502 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d4390a8db18d464faf12a922a397402a 2024-11-08T10:28:38,522 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/d4390a8db18d464faf12a922a397402a is 205, key is testBlockEvictionOnRegionMove1731061715986,,1731061716001.48e478bc9e54df81ffb6bf52e5f85fdf./info:regioninfo/1731061718385/Put/seqid=0 2024-11-08T10:28:38,523 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/d4390a8db18d464faf12a922a397402a 2024-11-08T10:28:38,534 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 639034ed9a40412ba7f4220c70e5a54e 2024-11-08T10:28:38,542 INFO [regionserver/0e58e25f69ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:38,542 INFO [regionserver/0e58e25f69ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:38,550 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/639034ed9a40412ba7f4220c70e5a54e is 43, key is default/ns:d/1731061715779/Put/seqid=0 2024-11-08T10:28:38,550 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/639034ed9a40412ba7f4220c70e5a54e 2024-11-08T10:28:38,561 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7917704086184d7aa48361fbba3e2fd6 2024-11-08T10:28:38,574 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/7917704086184d7aa48361fbba3e2fd6 is 78, key is testBlockEvictionOnRegionMove1731061715986/table:state/1731061716454/Put/seqid=0 2024-11-08T10:28:38,575 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/7917704086184d7aa48361fbba3e2fd6 2024-11-08T10:28:38,578 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/d4390a8db18d464faf12a922a397402a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/d4390a8db18d464faf12a922a397402a 2024-11-08T10:28:38,579 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/d4390a8db18d464faf12a922a397402a, delay=950 ms 2024-11-08T10:28:38,581 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/d4390a8db18d464faf12a922a397402a, entries=15, sequenceid=15, filesize=7.6 K 2024-11-08T10:28:38,582 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/639034ed9a40412ba7f4220c70e5a54e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/639034ed9a40412ba7f4220c70e5a54e 2024-11-08T10:28:38,583 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/639034ed9a40412ba7f4220c70e5a54e, delay=977 ms 2024-11-08T10:28:38,584 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/639034ed9a40412ba7f4220c70e5a54e, entries=2, sequenceid=15, filesize=5.0 K 2024-11-08T10:28:38,585 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/7917704086184d7aa48361fbba3e2fd6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/7917704086184d7aa48361fbba3e2fd6 2024-11-08T10:28:38,586 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/7917704086184d7aa48361fbba3e2fd6, delay=921 ms 2024-11-08T10:28:38,587 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/7917704086184d7aa48361fbba3e2fd6, entries=2, sequenceid=15, filesize=5.3 K 2024-11-08T10:28:38,588 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 125ms, sequenceid=15, compaction requested=false 2024-11-08T10:28:38,589 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T10:28:38,590 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/d4390a8db18d464faf12a922a397402a 2024-11-08T10:28:38,592 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/639034ed9a40412ba7f4220c70e5a54e 2024-11-08T10:28:38,593 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/7917704086184d7aa48361fbba3e2fd6 2024-11-08T10:28:38,598 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-08T10:28:38,599 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:28:38,599 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:38,600 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731061718461Running coprocessor pre-close hooks at 1731061718462 (+1 ms)Disabling compacts and flushes for region at 1731061718462Disabling writes for close at 1731061718463 (+1 ms)Obtaining lock to block concurrent updates at 1731061718463Preparing flush snapshotting stores in 1588230740 at 1731061718463Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731061718464 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731061718464Flushing 1588230740/info: creating writer at 1731061718465 (+1 ms)Flushing 1588230740/info: appending metadata at 1731061718520 (+55 ms)Flushing 1588230740/info: closing flushed file at 1731061718521 (+1 ms)Flushing 1588230740/ns: creating writer at 1731061718527 (+6 ms)Flushing 1588230740/ns: appending metadata at 1731061718548 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731061718549 (+1 ms)Flushing 1588230740/table: creating writer at 1731061718553 (+4 ms)Flushing 1588230740/table: appending metadata at 1731061718574 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731061718574Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@745e74cc: reopening flushed file at 1731061718577 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a2aece9: reopening flushed file at 1731061718581 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f037323: reopening flushed file at 1731061718585 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 125ms, sequenceid=15, compaction requested=false at 1731061718589 (+4 ms)Writing region close event to WAL at 1731061718593 (+4 ms)Running coprocessor post-close hooks at 1731061718599 (+6 ms)Closed at 1731061718599 2024-11-08T10:28:38,600 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:38,662 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(976): stopping server 0e58e25f69ec,34437,1731061713391; all regions closed. 2024-11-08T10:28:38,670 DEBUG [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:38,670 INFO [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C34437%2C1731061713391.meta:.meta(num 1731061715617) 2024-11-08T10:28:38,673 DEBUG [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:38,673 INFO [RS:0;0e58e25f69ec:34437 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C34437%2C1731061713391:(num 1731061715197) 2024-11-08T10:28:38,673 DEBUG [RS:0;0e58e25f69ec:34437 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,673 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:38,673 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:38,674 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.ChoreService(370): Chore service for: regionserver/0e58e25f69ec:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:38,674 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:38,674 INFO [regionserver/0e58e25f69ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:38,675 INFO [RS:0;0e58e25f69ec:34437 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34437 2024-11-08T10:28:38,688 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(976): stopping server 0e58e25f69ec,39659,1731061713537; all regions closed. 2024-11-08T10:28:38,691 DEBUG [RS:1;0e58e25f69ec:39659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:38,691 INFO [RS:1;0e58e25f69ec:39659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C39659%2C1731061713537:(num 1731061715199) 2024-11-08T10:28:38,691 DEBUG [RS:1;0e58e25f69ec:39659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:38,691 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:38,691 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:38,691 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.ChoreService(370): Chore service for: regionserver/0e58e25f69ec:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:38,692 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:28:38,692 INFO [regionserver/0e58e25f69ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:38,692 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:28:38,692 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:28:38,692 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:38,692 INFO [RS:1;0e58e25f69ec:39659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39659 2024-11-08T10:28:38,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:38,702 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0e58e25f69ec,34437,1731061713391 2024-11-08T10:28:38,702 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:38,713 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0e58e25f69ec,39659,1731061713537 2024-11-08T10:28:38,713 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:38,722 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0e58e25f69ec,34437,1731061713391] 2024-11-08T10:28:38,738 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0e58e25f69ec,34437,1731061713391 already deleted, retry=false 2024-11-08T10:28:38,738 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0e58e25f69ec,34437,1731061713391 expired; onlineServers=1 2024-11-08T10:28:38,738 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0e58e25f69ec,39659,1731061713537] 2024-11-08T10:28:38,746 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0e58e25f69ec,39659,1731061713537 already deleted, retry=false 2024-11-08T10:28:38,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0e58e25f69ec,39659,1731061713537 expired; onlineServers=0 2024-11-08T10:28:38,747 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0e58e25f69ec,39745,1731061712578' ***** 2024-11-08T10:28:38,747 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T10:28:38,747 INFO [M:0;0e58e25f69ec:39745 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:38,747 INFO [M:0;0e58e25f69ec:39745 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:38,747 DEBUG [M:0;0e58e25f69ec:39745 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T10:28:38,747 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T10:28:38,747 DEBUG [M:0;0e58e25f69ec:39745 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T10:28:38,748 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061714839 {}] cleaner.HFileCleaner(306): Exit Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061714839,5,FailOnTimeoutGroup] 2024-11-08T10:28:38,748 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061714827 {}] cleaner.HFileCleaner(306): Exit Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061714827,5,FailOnTimeoutGroup] 2024-11-08T10:28:38,748 INFO [M:0;0e58e25f69ec:39745 {}] hbase.ChoreService(370): Chore service for: master/0e58e25f69ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:38,748 INFO [M:0;0e58e25f69ec:39745 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:38,748 DEBUG [M:0;0e58e25f69ec:39745 {}] master.HMaster(1795): Stopping service threads 2024-11-08T10:28:38,748 INFO [M:0;0e58e25f69ec:39745 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T10:28:38,748 INFO [M:0;0e58e25f69ec:39745 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T10:28:38,749 INFO [M:0;0e58e25f69ec:39745 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T10:28:38,749 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T10:28:38,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:38,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:38,755 DEBUG [M:0;0e58e25f69ec:39745 {}] zookeeper.ZKUtil(347): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T10:28:38,756 WARN [M:0;0e58e25f69ec:39745 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T10:28:38,756 INFO [M:0;0e58e25f69ec:39745 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.lastflushedseqids 2024-11-08T10:28:38,767 INFO [M:0;0e58e25f69ec:39745 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T10:28:38,768 INFO [M:0;0e58e25f69ec:39745 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T10:28:38,768 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:28:38,768 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:38,768 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:38,768 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:28:38,768 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:38,769 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-08T10:28:38,780 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2a985ef8e060410f9c91e574a7914f26 2024-11-08T10:28:38,801 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a985ef8e060410f9c91e574a7914f26 is 82, key is hbase:meta,,1/info:regioninfo/1731061715689/Put/seqid=0 2024-11-08T10:28:38,802 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a985ef8e060410f9c91e574a7914f26 2024-11-08T10:28:38,813 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4245b3ee42a14d26a2f5e54f2a434cc7 2024-11-08T10:28:38,822 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:38,822 DEBUG [pool-14-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34437-0x1011b4718660001, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:38,823 INFO [RS:0;0e58e25f69ec:34437 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:38,823 INFO [RS:0;0e58e25f69ec:34437 {}] regionserver.HRegionServer(1031): Exiting; stopping=0e58e25f69ec,34437,1731061713391; zookeeper connection closed. 2024-11-08T10:28:38,824 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3e916184 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3e916184 2024-11-08T10:28:38,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:38,830 INFO [RS:1;0e58e25f69ec:39659 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:38,830 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39659-0x1011b4718660002, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:38,830 INFO [RS:1;0e58e25f69ec:39659 {}] regionserver.HRegionServer(1031): Exiting; stopping=0e58e25f69ec,39659,1731061713537; zookeeper connection closed. 2024-11-08T10:28:38,831 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2dcbef4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2dcbef4 2024-11-08T10:28:38,831 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T10:28:38,833 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4245b3ee42a14d26a2f5e54f2a434cc7 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731061716460/Put/seqid=0 2024-11-08T10:28:38,834 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4245b3ee42a14d26a2f5e54f2a434cc7 2024-11-08T10:28:38,845 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 165b4e604105486f85da2f8a63cb1017 2024-11-08T10:28:38,864 DEBUG [M:0;0e58e25f69ec:39745 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/165b4e604105486f85da2f8a63cb1017 is 69, key is 0e58e25f69ec,34437,1731061713391/rs:state/1731061714872/Put/seqid=0 2024-11-08T10:28:38,865 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/165b4e604105486f85da2f8a63cb1017 2024-11-08T10:28:38,868 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2a985ef8e060410f9c91e574a7914f26 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2a985ef8e060410f9c91e574a7914f26 2024-11-08T10:28:38,870 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2a985ef8e060410f9c91e574a7914f26, entries=8, sequenceid=84, filesize=5.5 K 2024-11-08T10:28:38,871 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/4245b3ee42a14d26a2f5e54f2a434cc7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4245b3ee42a14d26a2f5e54f2a434cc7 2024-11-08T10:28:38,874 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/4245b3ee42a14d26a2f5e54f2a434cc7, entries=9, sequenceid=84, filesize=6.5 K 2024-11-08T10:28:38,875 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/165b4e604105486f85da2f8a63cb1017 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/165b4e604105486f85da2f8a63cb1017 2024-11-08T10:28:38,877 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/165b4e604105486f85da2f8a63cb1017, entries=2, sequenceid=84, filesize=5.1 K 2024-11-08T10:28:38,879 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=84, compaction requested=false 2024-11-08T10:28:38,881 INFO [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:38,881 DEBUG [M:0;0e58e25f69ec:39745 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731061718768Disabling compacts and flushes for region at 1731061718768Disabling writes for close at 1731061718768Obtaining lock to block concurrent updates at 1731061718769 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731061718769Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731061718770 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731061718770Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731061718770Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731061718800 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731061718800Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731061718805 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731061718832 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731061718832Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731061718837 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731061718863 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731061718863Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf02c01: reopening flushed file at 1731061718867 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ea1628: reopening flushed file at 1731061718871 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac9ad73: reopening flushed file at 1731061718874 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 110ms, sequenceid=84, compaction requested=false at 1731061718879 (+5 ms)Writing region close event to WAL at 1731061718881 (+2 ms)Closed at 1731061718881 2024-11-08T10:28:38,883 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:38,883 INFO [M:0;0e58e25f69ec:39745 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T10:28:38,884 INFO [M:0;0e58e25f69ec:39745 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39745 2024-11-08T10:28:38,884 INFO [M:0;0e58e25f69ec:39745 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:39,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:39,013 INFO [M:0;0e58e25f69ec:39745 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:39,013 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39745-0x1011b4718660000, quorum=127.0.0.1:51368, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:39,019 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T10:28:39,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T10:28:39,048 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,058 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0e58e25f69ec:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731061716828 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/0e58e25f69ec:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731061718580 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/0e58e25f69ec:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/0e58e25f69ec:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731061718583 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731061718364 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=447 (was 443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5462 (was 5835) 2024-11-08T10:28:39,066 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=447, ProcessCount=11, AvailableMemoryMB=5462 2024-11-08T10:28:39,066 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/hadoop-log-dir so I do NOT create it in target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c 2024-11-08T10:28:39,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/hadoop-tmp-dir so I do NOT create it in target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c 2024-11-08T10:28:39,067 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,070 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/cluster_d11af024-ebcc-2595-17ea-abaa46c87fcf, deleteOnExit=true 2024-11-08T10:28:39,074 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/cluster_d11af024-ebcc-2595-17ea-abaa46c87fcf/zookeeper_0, clientPort=53173, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/cluster_d11af024-ebcc-2595-17ea-abaa46c87fcf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/cluster_d11af024-ebcc-2595-17ea-abaa46c87fcf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T10:28:39,075 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53173 2024-11-08T10:28:39,087 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 with version=8 2024-11-08T10:28:39,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase-staging 2024-11-08T10:28:39,090 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:39,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:39,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,091 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:39,091 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T10:28:39,091 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:39,092 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42311 2024-11-08T10:28:39,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42311 connecting to ZooKeeper ensemble=127.0.0.1:53173 2024-11-08T10:28:39,132 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423110x0, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:39,133 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42311-0x1011b4732e60000 connected 2024-11-08T10:28:39,198 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,199 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,200 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:39,200 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582, hbase.cluster.distributed=false 2024-11-08T10:28:39,203 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:39,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42311 2024-11-08T10:28:39,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42311 2024-11-08T10:28:39,210 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42311 2024-11-08T10:28:39,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42311 2024-11-08T10:28:39,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42311 2024-11-08T10:28:39,230 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:39,230 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:28:39,231 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:39,232 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40963 2024-11-08T10:28:39,233 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40963 connecting to ZooKeeper ensemble=127.0.0.1:53173 2024-11-08T10:28:39,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,247 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:409630x0, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:39,248 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40963-0x1011b4732e60001 connected 2024-11-08T10:28:39,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:39,249 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:28:39,249 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache 2024-11-08T10:28:39,249 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T10:28:39,249 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T10:28:39,249 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T10:28:39,250 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T10:28:39,250 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T10:28:39,250 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T10:28:39,251 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T10:28:39,251 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence is complete. 2024-11-08T10:28:39,255 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:28:39,256 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:28:39,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:39,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40963 2024-11-08T10:28:39,262 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40963 2024-11-08T10:28:39,263 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40963 2024-11-08T10:28:39,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40963 2024-11-08T10:28:39,266 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40963 2024-11-08T10:28:39,294 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:39,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,294 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:39,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:39,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:39,295 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:28:39,295 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:39,296 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45305 2024-11-08T10:28:39,299 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45305 connecting to ZooKeeper ensemble=127.0.0.1:53173 2024-11-08T10:28:39,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,299 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453050x0, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:39,425 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453050x0, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:39,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45305-0x1011b4732e60002 connected 2024-11-08T10:28:39,426 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:28:39,426 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache 2024-11-08T10:28:39,426 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T10:28:39,427 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T10:28:39,427 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T10:28:39,427 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T10:28:39,427 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T10:28:39,427 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T10:28:39,427 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T10:28:39,427 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence is complete. 2024-11-08T10:28:39,427 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:28:39,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T10:28:39,430 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:39,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45305 2024-11-08T10:28:39,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45305 2024-11-08T10:28:39,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45305 2024-11-08T10:28:39,435 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45305 2024-11-08T10:28:39,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45305 2024-11-08T10:28:39,451 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0e58e25f69ec:42311 2024-11-08T10:28:39,452 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,464 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,464 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,473 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:39,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,473 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:39,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,475 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T10:28:39,475 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0e58e25f69ec,42311,1731061719089 from backup master directory 2024-11-08T10:28:39,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,481 WARN [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:39,481 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T10:28:39,482 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,483 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase.id] with ID: d6f814d7-71c0-4b5e-9eed-0ebd5764f5b4 2024-11-08T10:28:39,483 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.tmp/hbase.id 2024-11-08T10:28:39,487 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/hbase.id] 2024-11-08T10:28:39,525 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:39,525 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T10:28:39,528 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-08T10:28:39,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,540 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,544 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:28:39,545 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T10:28:39,546 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:39,562 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store 2024-11-08T10:28:39,568 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:39,568 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:28:39,568 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:39,568 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:39,568 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:28:39,568 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:39,568 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:39,569 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731061719568Disabling compacts and flushes for region at 1731061719568Disabling writes for close at 1731061719568Writing region close event to WAL at 1731061719568Closed at 1731061719568 2024-11-08T10:28:39,569 WARN [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/.initializing 2024-11-08T10:28:39,569 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,574 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:39,577 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0e58e25f69ec%2C42311%2C1731061719089, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,42311,1731061719089, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/oldWALs, maxLogs=10 2024-11-08T10:28:39,594 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/WALs/0e58e25f69ec,42311,1731061719089/0e58e25f69ec%2C42311%2C1731061719089.1731061719578 2024-11-08T10:28:39,594 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:39,594 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:39,595 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:39,595 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,595 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,595 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,600 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T10:28:39,601 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:39,601 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T10:28:39,606 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:39,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T10:28:39,610 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:39,610 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T10:28:39,614 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:39,614 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,615 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,615 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,615 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,616 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,616 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:39,616 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T10:28:39,623 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:39,623 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73165116, jitterRate=0.09024518728256226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:39,624 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731061719595Initializing all the Stores at 1731061719595Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061719595Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061719595Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061719595Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061719595Cleaning up temporary data from old regions at 1731061719616 (+21 ms)Region opened successfully at 1731061719624 (+8 ms) 2024-11-08T10:28:39,624 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T10:28:39,628 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad8f613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:39,629 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T10:28:39,629 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T10:28:39,629 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T10:28:39,629 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T10:28:39,629 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T10:28:39,630 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T10:28:39,630 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T10:28:39,632 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T10:28:39,633 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T10:28:39,648 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T10:28:39,649 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T10:28:39,650 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T10:28:39,656 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T10:28:39,657 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T10:28:39,658 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T10:28:39,664 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T10:28:39,666 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T10:28:39,673 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T10:28:39,676 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T10:28:39,681 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T10:28:39,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:39,690 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:39,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,690 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:39,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,690 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0e58e25f69ec,42311,1731061719089, sessionid=0x1011b4732e60000, setting cluster-up flag (Was=false) 2024-11-08T10:28:39,706 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,731 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T10:28:39,733 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,748 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:39,773 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T10:28:39,775 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:39,775 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T10:28:39,777 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:39,777 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T10:28:39,777 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T10:28:39,778 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0e58e25f69ec,42311,1731061719089 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T10:28:39,779 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:39,779 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=5, maxPoolSize=5 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0e58e25f69ec:0, corePoolSize=10, maxPoolSize=10 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:39,780 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731061749781 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T10:28:39,781 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T10:28:39,782 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,782 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:39,782 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T10:28:39,782 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T10:28:39,782 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T10:28:39,782 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T10:28:39,782 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T10:28:39,783 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T10:28:39,783 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061719783,5,FailOnTimeoutGroup] 2024-11-08T10:28:39,783 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061719783,5,FailOnTimeoutGroup] 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,783 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,799 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T10:28:39,800 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:39,805 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:39,806 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T10:28:39,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:28:39,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:39,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T10:28:39,814 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T10:28:39,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:39,815 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T10:28:39,818 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:28:39,818 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:39,818 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T10:28:39,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:28:39,821 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:39,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:39,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T10:28:39,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:39,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:39,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T10:28:39,823 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T10:28:39,823 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:39,824 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T10:28:39,829 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:39,830 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71215006, jitterRate=0.061186283826828}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:39,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731061719805Initializing all the Stores at 1731061719805Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061719806 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061719806Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061719806Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061719806Cleaning up temporary data from old regions at 1731061719823 (+17 ms)Region opened successfully at 1731061719830 (+7 ms) 2024-11-08T10:28:39,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:28:39,831 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T10:28:39,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T10:28:39,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:28:39,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:28:39,832 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:39,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731061719831Disabling compacts and flushes for region at 1731061719831Disabling writes for close at 1731061719831Writing region close event to WAL at 1731061719832 (+1 ms)Closed at 1731061719832 2024-11-08T10:28:39,833 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:39,833 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T10:28:39,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T10:28:39,834 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:39,834 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T10:28:39,839 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(746): ClusterId : d6f814d7-71c0-4b5e-9eed-0ebd5764f5b4 2024-11-08T10:28:39,839 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(746): ClusterId : d6f814d7-71c0-4b5e-9eed-0ebd5764f5b4 2024-11-08T10:28:39,839 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:28:39,839 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:28:39,857 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:28:39,857 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:28:39,857 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:28:39,857 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:28:39,865 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:28:39,865 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:28:39,866 DEBUG [RS:1;0e58e25f69ec:45305 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351a69d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:39,866 DEBUG [RS:0;0e58e25f69ec:40963 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29f0a97b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:39,878 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0e58e25f69ec:40963 2024-11-08T10:28:39,878 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T10:28:39,878 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T10:28:39,878 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T10:28:39,879 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(2659): reportForDuty to master=0e58e25f69ec,42311,1731061719089 with port=40963, startcode=1731061719230 2024-11-08T10:28:39,880 DEBUG [RS:0;0e58e25f69ec:40963 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:28:39,882 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:28:39,883 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,883 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(517): Registering regionserver=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,884 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;0e58e25f69ec:45305 2024-11-08T10:28:39,884 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T10:28:39,884 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T10:28:39,884 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T10:28:39,885 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:39,885 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T10:28:39,885 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(2659): reportForDuty to master=0e58e25f69ec,42311,1731061719089 with port=45305, startcode=1731061719293 2024-11-08T10:28:39,885 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T10:28:39,885 DEBUG [RS:1;0e58e25f69ec:45305 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:28:39,889 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47951, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:28:39,890 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,890 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(517): Registering regionserver=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,892 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:39,892 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T10:28:39,892 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T10:28:39,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:39,930 DEBUG [RS:0;0e58e25f69ec:40963 {}] zookeeper.ZKUtil(111): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,930 WARN [RS:0;0e58e25f69ec:40963 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:39,930 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0e58e25f69ec,40963,1731061719230] 2024-11-08T10:28:39,931 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0e58e25f69ec,45305,1731061719293] 2024-11-08T10:28:39,931 INFO [RS:0;0e58e25f69ec:40963 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:39,931 DEBUG [RS:1;0e58e25f69ec:45305 {}] zookeeper.ZKUtil(111): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,931 WARN [RS:1;0e58e25f69ec:45305 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:39,931 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,931 INFO [RS:1;0e58e25f69ec:45305 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:39,931 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,936 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:28:39,936 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:28:39,938 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:28:39,938 INFO [RS:0;0e58e25f69ec:40963 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:28:39,938 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,942 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T10:28:39,943 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:28:39,944 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T10:28:39,944 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,944 INFO [RS:1;0e58e25f69ec:45305 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,944 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,944 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,944 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,945 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:39,945 DEBUG [RS:0;0e58e25f69ec:40963 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:39,945 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,945 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,40963,1731061719230-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,946 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,947 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,947 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,947 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:39,947 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:39,947 DEBUG [RS:1;0e58e25f69ec:45305 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,948 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,45305,1731061719293-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:39,961 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:28:39,961 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,40963,1731061719230-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,961 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,961 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.Replication(171): 0e58e25f69ec,40963,1731061719230 started 2024-11-08T10:28:39,962 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:28:39,962 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,45305,1731061719293-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,962 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,962 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.Replication(171): 0e58e25f69ec,45305,1731061719293 started 2024-11-08T10:28:39,976 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,976 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1482): Serving as 0e58e25f69ec,45305,1731061719293, RpcServer on 0e58e25f69ec/172.17.0.2:45305, sessionid=0x1011b4732e60002 2024-11-08T10:28:39,976 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:28:39,976 DEBUG [RS:1;0e58e25f69ec:45305 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,976 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,45305,1731061719293' 2024-11-08T10:28:39,976 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:28:39,976 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:39,976 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1482): Serving as 0e58e25f69ec,40963,1731061719230, RpcServer on 0e58e25f69ec/172.17.0.2:40963, sessionid=0x1011b4732e60001 2024-11-08T10:28:39,976 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:28:39,976 DEBUG [RS:0;0e58e25f69ec:40963 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,976 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,40963,1731061719230' 2024-11-08T10:28:39,977 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:28:39,977 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:28:39,977 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:28:39,977 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:28:39,977 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:28:39,978 DEBUG [RS:1;0e58e25f69ec:45305 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:39,978 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,45305,1731061719293' 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:28:39,978 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,40963,1731061719230' 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:28:39,978 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:28:39,978 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:28:39,978 DEBUG [RS:1;0e58e25f69ec:45305 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:28:39,978 INFO [RS:1;0e58e25f69ec:45305 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:28:39,978 INFO [RS:1;0e58e25f69ec:45305 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:28:39,979 DEBUG [RS:0;0e58e25f69ec:40963 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:28:39,979 INFO [RS:0;0e58e25f69ec:40963 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:28:39,979 INFO [RS:0;0e58e25f69ec:40963 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:28:39,985 WARN [0e58e25f69ec:42311 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T10:28:40,079 INFO [RS:1;0e58e25f69ec:45305 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:40,079 INFO [RS:0;0e58e25f69ec:40963 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:40,080 INFO [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C45305%2C1731061719293, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:40,080 INFO [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C40963%2C1731061719230, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,40963,1731061719230, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:40,109 INFO [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293/0e58e25f69ec%2C45305%2C1731061719293.1731061720083 2024-11-08T10:28:40,110 DEBUG [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:40,112 INFO [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,40963,1731061719230/0e58e25f69ec%2C40963%2C1731061719230.1731061720083 2024-11-08T10:28:40,112 DEBUG [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:40,235 DEBUG [0e58e25f69ec:42311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T10:28:40,236 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(204): Hosts are {0e58e25f69ec=0} racks are {/default-rack=0} 2024-11-08T10:28:40,237 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T10:28:40,237 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T10:28:40,237 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T10:28:40,237 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T10:28:40,237 INFO [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T10:28:40,237 INFO [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T10:28:40,237 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T10:28:40,238 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:40,239 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,45305,1731061719293, state=OPENING 2024-11-08T10:28:40,273 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T10:28:40,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:40,281 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:40,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:40,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,283 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:40,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,45305,1731061719293}] 2024-11-08T10:28:40,436 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T10:28:40,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44643, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T10:28:40,440 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T10:28:40,441 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:40,441 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T10:28:40,442 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C45305%2C1731061719293.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:40,460 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293/0e58e25f69ec%2C45305%2C1731061719293.meta.1731061720442.meta 2024-11-08T10:28:40,462 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:40,462 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T10:28:40,463 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T10:28:40,463 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T10:28:40,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T10:28:40,467 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:28:40,467 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,467 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:40,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T10:28:40,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T10:28:40,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:40,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T10:28:40,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:28:40,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:40,470 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T10:28:40,471 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:28:40,471 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,471 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:40,471 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T10:28:40,472 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:40,472 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:40,472 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T10:28:40,473 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T10:28:40,473 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:40,473 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T10:28:40,474 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75478815, jitterRate=0.12472198903560638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:40,474 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T10:28:40,474 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731061720464Writing region info on filesystem at 1731061720464Initializing all the Stores at 1731061720464Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061720464Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061720466 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061720466Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061720466Cleaning up temporary data from old regions at 1731061720473 (+7 ms)Running coprocessor post-open hooks at 1731061720474 (+1 ms)Region opened successfully at 1731061720474 2024-11-08T10:28:40,475 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731061720436 2024-11-08T10:28:40,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:40,479 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,45305,1731061719293, state=OPEN 2024-11-08T10:28:40,482 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T10:28:40,482 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T10:28:40,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:40,490 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:40,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:40,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,490 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:40,490 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:40,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T10:28:40,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,45305,1731061719293 in 207 msec 2024-11-08T10:28:40,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T10:28:40,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 660 msec 2024-11-08T10:28:40,496 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T10:28:40,496 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T10:28:40,502 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T10:28:40,502 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1] 2024-11-08T10:28:40,502 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:28:40,504 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38325, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:28:40,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 733 msec 2024-11-08T10:28:40,510 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731061720510, completionTime=-1 2024-11-08T10:28:40,511 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T10:28:40,511 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T10:28:40,513 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T10:28:40,513 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731061780513 2024-11-08T10:28:40,513 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731061840513 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0e58e25f69ec:42311, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,514 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,516 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,518 DEBUG [master/0e58e25f69ec:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T10:28:40,521 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.039sec 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:40,522 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T10:28:40,523 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T10:28:40,523 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T10:28:40,524 INFO [master/0e58e25f69ec:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,42311,1731061719089-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:40,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e7db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:40,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0e58e25f69ec,42311,-1 for getting cluster id 2024-11-08T10:28:40,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T10:28:40,543 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd6f814d7-71c0-4b5e-9eed-0ebd5764f5b4' 2024-11-08T10:28:40,544 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T10:28:40,544 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d6f814d7-71c0-4b5e-9eed-0ebd5764f5b4" 2024-11-08T10:28:40,545 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae364a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:40,545 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0e58e25f69ec,42311,-1] 2024-11-08T10:28:40,545 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T10:28:40,550 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:40,552 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T10:28:40,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T10:28:40,555 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T10:28:40,557 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1] 2024-11-08T10:28:40,558 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:28:40,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53888, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:28:40,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:40,565 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T10:28:40,568 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:40,568 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@26336bf0 2024-11-08T10:28:40,569 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T10:28:40,572 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T10:28:40,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731061720565', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T10:28:40,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 2024-11-08T10:28:40,577 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T10:28:40,578 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731061720565" procId is: 4 2024-11-08T10:28:40,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T10:28:40,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:40,600 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 437c85594b3e91246859c460e224f047, NAME => 'testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731061720565', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:40,611 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:40,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1722): Closing 437c85594b3e91246859c460e224f047, disabling compactions & flushes 2024-11-08T10:28:40,612 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. after waiting 0 ms 2024-11-08T10:28:40,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,612 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,612 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731061720565-pool-0 {}] regionserver.HRegion(1676): Region close journal for 437c85594b3e91246859c460e224f047: Waiting for close lock at 1731061720612Disabling compacts and flushes for region at 1731061720612Disabling writes for close at 1731061720612Writing region close event to WAL at 1731061720612Closed at 1731061720612 2024-11-08T10:28:40,614 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T10:28:40,614 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731061720614"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731061720614"}]},"ts":"1731061720614"} 2024-11-08T10:28:40,618 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T10:28:40,619 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T10:28:40,620 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731061720565","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061720619"}]},"ts":"1731061720619"} 2024-11-08T10:28:40,623 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731061720565, state=ENABLING in hbase:meta 2024-11-08T10:28:40,623 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {0e58e25f69ec=0} racks are {/default-rack=0} 2024-11-08T10:28:40,624 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T10:28:40,624 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T10:28:40,624 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T10:28:40,624 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T10:28:40,624 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T10:28:40,624 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T10:28:40,624 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T10:28:40,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN}] 2024-11-08T10:28:40,627 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN 2024-11-08T10:28:40,628 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN; state=OFFLINE, location=0e58e25f69ec,45305,1731061719293; forceNewPlan=false, retain=false 2024-11-08T10:28:40,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:40,779 INFO [0e58e25f69ec:42311 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T10:28:40,780 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=437c85594b3e91246859c460e224f047, regionState=OPENING, regionLocation=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:40,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN because future has completed 2024-11-08T10:28:40,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,45305,1731061719293}] 2024-11-08T10:28:40,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:40,939 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,940 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 437c85594b3e91246859c460e224f047, NAME => 'testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:40,940 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731061720565 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,940 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:40,940 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,940 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,941 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,945 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437c85594b3e91246859c460e224f047 columnFamilyName family 2024-11-08T10:28:40,945 DEBUG [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:40,945 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] regionserver.HStore(327): Store=437c85594b3e91246859c460e224f047/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:40,945 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,946 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,946 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,947 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,947 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,947 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,953 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T10:28:40,953 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 437c85594b3e91246859c460e224f047; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65408565, jitterRate=-0.025336429476737976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:28:40,953 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:40,954 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 437c85594b3e91246859c460e224f047: Running coprocessor pre-open hook at 1731061720941Writing region info on filesystem at 1731061720941Initializing all the Stores at 1731061720941Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061720941Cleaning up temporary data from old regions at 1731061720947 (+6 ms)Running coprocessor post-open hooks at 1731061720953 (+6 ms)Region opened successfully at 1731061720954 (+1 ms) 2024-11-08T10:28:40,954 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047., pid=6, masterSystemTime=1731061720936 2024-11-08T10:28:40,957 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,957 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:40,957 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=437c85594b3e91246859c460e224f047, regionState=OPEN, openSeqNum=2, regionLocation=0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:40,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,45305,1731061719293 because future has completed 2024-11-08T10:28:40,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T10:28:40,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,45305,1731061719293 in 177 msec 2024-11-08T10:28:40,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T10:28:40,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN in 338 msec 2024-11-08T10:28:40,965 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T10:28:40,965 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731061720565","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731061720965"}]},"ts":"1731061720965"} 2024-11-08T10:28:40,967 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731061720565, state=ENABLED in hbase:meta 2024-11-08T10:28:40,968 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T10:28:40,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731061720565 in 394 msec 2024-11-08T10:28:41,158 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T10:28:41,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42311 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T10:28:41,208 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731061720565 completed 2024-11-08T10:28:41,208 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731061720565 get assigned. Timeout = 60000ms 2024-11-08T10:28:41,208 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:28:41,212 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731061720565 assigned to meta. Checking AM states. 2024-11-08T10:28:41,212 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T10:28:41,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731061720565 assigned. 2024-11-08T10:28:41,217 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731061720565', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047., hostname=0e58e25f69ec,45305,1731061719293, seqNum=2] 2024-11-08T10:28:41,223 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 437c85594b3e91246859c460e224f047 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T10:28:41,233 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:41,245 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/.tmp/family/9002343d382740e4a18e8511bcd0ac4c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T10:28:41,246 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/.tmp/family/9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:41,248 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/.tmp/family/9002343d382740e4a18e8511bcd0ac4c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:41,249 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, delay=980 ms 2024-11-08T10:28:41,250 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T10:28:41,251 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 437c85594b3e91246859c460e224f047 in 28ms, sequenceid=6, compaction requested=false 2024-11-08T10:28:41,251 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731061720565' 2024-11-08T10:28:41,252 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 437c85594b3e91246859c460e224f047: 2024-11-08T10:28:42,230 DEBUG [hfile-prefetch-1731061716828 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9002343d382740e4a18e8511bcd0ac4c_0. 2024-11-08T10:28:42,231 DEBUG [hfile-prefetch-1731061716828 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9002343d382740e4a18e8511bcd0ac4c_117. 2024-11-08T10:28:42,231 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, with total blocks 2, and data blocks 1 2024-11-08T10:28:42,231 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T10:28:42,231 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T10:28:42,231 DEBUG [hfile-prefetch-1731061716828 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c. Will sleep 100ms and try the verification again. 2024-11-08T10:28:42,252 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;0e58e25f69ec:45305,5,FailOnTimeoutGroup] 2024-11-08T10:28:42,252 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0e58e25f69ec,45305,1731061719293' ***** 2024-11-08T10:28:42,252 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-08T10:28:42,253 INFO [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T10:28:42,300 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T10:28:42,300 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T10:28:42,300 INFO [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence 2024-11-08T10:28:42,300 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T10:28:42,301 INFO [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T10:28:42,301 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T10:28:42,311 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1488): Thread RS:1;0e58e25f69ec:45305 started persisting bucket cache to file 2024-11-08T10:28:42,311 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:42,311 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:42,317 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T10:28:42,317 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731061722311 2024-11-08T10:28:42,317 DEBUG [RS:1;0e58e25f69ec:45305 {}] bucket.BucketCache(1510): Thread RS:1;0e58e25f69ec:45305 finished persisting bucket cache to file, renaming 2024-11-08T10:28:42,317 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:28:42,317 INFO [RS:1;0e58e25f69ec:45305 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:28:42,317 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T10:28:42,317 INFO [RS:1;0e58e25f69ec:45305 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:28:42,317 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(3091): Received CLOSE for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(959): stopping server 0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;0e58e25f69ec:45305. 2024-11-08T10:28:42,318 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:42,318 DEBUG [RS:1;0e58e25f69ec:45305 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:42,318 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 437c85594b3e91246859c460e224f047, disabling compactions & flushes 2024-11-08T10:28:42,318 DEBUG [RS:1;0e58e25f69ec:45305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:42,318 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:42,318 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:28:42,318 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. after waiting 0 ms 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:28:42,318 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:28:42,318 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T10:28:42,319 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T10:28:42,319 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 437c85594b3e91246859c460e224f047=testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.} 2024-11-08T10:28:42,319 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:42,319 DEBUG [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 437c85594b3e91246859c460e224f047 2024-11-08T10:28:42,319 DEBUG [StoreFileCloser-437c85594b3e91246859c460e224f047-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:42,319 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:28:42,319 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T10:28:42,319 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T10:28:42,319 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:28:42,319 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:28:42,319 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-08T10:28:42,321 WARN [hfile-prefetch-1731061716828 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T10:28:42,322 DEBUG [hfile-prefetch-1731061716828 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:42,324 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T10:28:42,324 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:42,325 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 437c85594b3e91246859c460e224f047: Waiting for close lock at 1731061722318Running coprocessor pre-close hooks at 1731061722318Disabling compacts and flushes for region at 1731061722318Disabling writes for close at 1731061722318Writing region close event to WAL at 1731061722319 (+1 ms)Running coprocessor post-close hooks at 1731061722324 (+5 ms)Closed at 1731061722324 2024-11-08T10:28:42,325 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:42,331 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:42,343 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/b4229a0d9caa463caf64f2358f6d23f2 is 209, key is testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047./info:regioninfo/1731061720957/Put/seqid=0 2024-11-08T10:28:42,344 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:42,350 INFO [regionserver/0e58e25f69ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:42,351 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:42,353 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T10:28:42,353 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T10:28:42,353 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:42,353 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:42,355 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:42,355 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061722353 2024-11-08T10:28:42,355 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T10:28:42,369 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/482791e400b845babc4e9d7ba71cc052 is 43, key is default/ns:d/1731061720505/Put/seqid=0 2024-11-08T10:28:42,370 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:42,378 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:42,389 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/9b5db6f579534ccd913c48a4e22d8556 is 80, key is testBlockEvictionOnGracefulStop1731061720565/table:state/1731061720965/Put/seqid=0 2024-11-08T10:28:42,390 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:42,392 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/b4229a0d9caa463caf64f2358f6d23f2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:42,393 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2, delay=928 ms 2024-11-08T10:28:42,394 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2, entries=10, sequenceid=11, filesize=7.0 K 2024-11-08T10:28:42,394 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/ns/482791e400b845babc4e9d7ba71cc052 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:42,395 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052, delay=930 ms 2024-11-08T10:28:42,396 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T10:28:42,397 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/table/9b5db6f579534ccd913c48a4e22d8556 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:42,398 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556, delay=973 ms 2024-11-08T10:28:42,399 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556, entries=2, sequenceid=11, filesize=5.3 K 2024-11-08T10:28:42,399 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false 2024-11-08T10:28:42,400 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:42,401 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:42,402 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:42,407 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T10:28:42,407 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:28:42,407 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:42,407 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731061722319Running coprocessor pre-close hooks at 1731061722319Disabling compacts and flushes for region at 1731061722319Disabling writes for close at 1731061722319Obtaining lock to block concurrent updates at 1731061722319Preparing flush snapshotting stores in 1588230740 at 1731061722319Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731061722320 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731061722320Flushing 1588230740/info: creating writer at 1731061722320Flushing 1588230740/info: appending metadata at 1731061722343 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731061722343Flushing 1588230740/ns: creating writer at 1731061722346 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731061722369 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1731061722369Flushing 1588230740/table: creating writer at 1731061722373 (+4 ms)Flushing 1588230740/table: appending metadata at 1731061722389 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731061722389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@39f7cd2f: reopening flushed file at 1731061722392 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3be9cdce: reopening flushed file at 1731061722394 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ffc565d: reopening flushed file at 1731061722396 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=11, compaction requested=false at 1731061722399 (+3 ms)Writing region close event to WAL at 1731061722403 (+4 ms)Running coprocessor post-close hooks at 1731061722407 (+4 ms)Closed at 1731061722407 2024-11-08T10:28:42,407 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:42,519 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(976): stopping server 0e58e25f69ec,45305,1731061719293; all regions closed. 2024-11-08T10:28:42,521 DEBUG [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:42,521 INFO [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C45305%2C1731061719293.meta:.meta(num 1731061720442) 2024-11-08T10:28:42,523 DEBUG [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:42,523 INFO [RS:1;0e58e25f69ec:45305 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C45305%2C1731061719293:(num 1731061720083) 2024-11-08T10:28:42,523 DEBUG [RS:1;0e58e25f69ec:45305 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:42,523 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:42,523 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:42,524 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.ChoreService(370): Chore service for: regionserver/0e58e25f69ec:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:42,524 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:42,524 INFO [regionserver/0e58e25f69ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:42,524 INFO [RS:1;0e58e25f69ec:45305 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45305 2024-11-08T10:28:42,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0e58e25f69ec,45305,1731061719293 2024-11-08T10:28:42,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:42,556 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:42,564 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0e58e25f69ec,45305,1731061719293] 2024-11-08T10:28:42,572 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0e58e25f69ec,45305,1731061719293 already deleted, retry=false 2024-11-08T10:28:42,572 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 0e58e25f69ec,45305,1731061719293 on 0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:42,576 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 0e58e25f69ec,45305,1731061719293, splitWal=true, meta=true 2024-11-08T10:28:42,579 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 0e58e25f69ec,45305,1731061719293, splitWal=true, meta=true 2024-11-08T10:28:42,579 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 0e58e25f69ec,45305,1731061719293 (carryingMeta=true) 0e58e25f69ec,45305,1731061719293/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1b0a6be3[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T10:28:42,580 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 0e58e25f69ec,45305,1731061719293, splitWal=true, meta=true, isMeta: true 2024-11-08T10:28:42,581 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293-splitting 2024-11-08T10:28:42,581 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293-splitting dir is empty, no logs to split. 2024-11-08T10:28:42,581 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0e58e25f69ec,45305,1731061719293 WAL count=0, meta=true 2024-11-08T10:28:42,583 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293-splitting dir is empty, no logs to split. 2024-11-08T10:28:42,583 INFO [PEWorker-2 {}] master.SplitWALManager(105): 0e58e25f69ec,45305,1731061719293 WAL count=0, meta=true 2024-11-08T10:28:42,584 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 0e58e25f69ec,45305,1731061719293 WAL splitting is done? wals=0, meta=true 2024-11-08T10:28:42,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T10:28:42,587 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:42,587 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T10:28:42,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:42,664 INFO [RS:1;0e58e25f69ec:45305 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:42,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45305-0x1011b4732e60002, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:42,664 INFO [RS:1;0e58e25f69ec:45305 {}] regionserver.HRegionServer(1031): Exiting; stopping=0e58e25f69ec,45305,1731061719293; zookeeper connection closed. 2024-11-08T10:28:42,665 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4708b3d1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4708b3d1 2024-11-08T10:28:42,738 DEBUG [0e58e25f69ec:42311 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T10:28:42,738 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:42,739 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,40963,1731061719230, state=OPENING 2024-11-08T10:28:42,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:42,764 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:42,764 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T10:28:42,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:42,764 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:42,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,40963,1731061719230}] 2024-11-08T10:28:42,775 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0e58e25f69ec:0 server-side Connection retries=45 2024-11-08T10:28:42,775 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:42,775 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:42,776 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T10:28:42,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T10:28:42,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T10:28:42,776 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T10:28:42,776 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T10:28:42,777 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36949 2024-11-08T10:28:42,779 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36949 connecting to ZooKeeper ensemble=127.0.0.1:53173 2024-11-08T10:28:42,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:42,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:42,789 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369490x0, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T10:28:42,790 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-08T10:28:42,790 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:369490x0, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-08T10:28:42,790 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36949-0x1011b4732e60003 connected 2024-11-08T10:28:42,790 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T10:28:42,791 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache 2024-11-08T10:28:42,791 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T10:28:42,791 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T10:28:42,791 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T10:28:42,795 INFO [Thread-207 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T10:28:42,795 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T10:28:42,796 INFO [Thread-207 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-08T10:28:42,796 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T10:28:42,796 INFO [Thread-207 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-08T10:28:42,797 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T10:28:42,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T10:28:42,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36949 2024-11-08T10:28:42,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36949 2024-11-08T10:28:42,799 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36949 2024-11-08T10:28:42,799 DEBUG [Thread-207 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-08T10:28:42,800 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T10:28:42,800 INFO [Thread-207 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-08T10:28:42,800 INFO [Thread-207 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence is complete. 2024-11-08T10:28:42,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36949 2024-11-08T10:28:42,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36949 2024-11-08T10:28:42,808 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(746): ClusterId : d6f814d7-71c0-4b5e-9eed-0ebd5764f5b4 2024-11-08T10:28:42,808 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T10:28:42,815 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T10:28:42,815 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T10:28:42,823 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T10:28:42,823 DEBUG [RS:2;0e58e25f69ec:36949 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6c5688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0e58e25f69ec/172.17.0.2:0 2024-11-08T10:28:42,835 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;0e58e25f69ec:36949 2024-11-08T10:28:42,835 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T10:28:42,835 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T10:28:42,835 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T10:28:42,836 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(2659): reportForDuty to master=0e58e25f69ec,42311,1731061719089 with port=36949, startcode=1731061722775 2024-11-08T10:28:42,836 DEBUG [RS:2;0e58e25f69ec:36949 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T10:28:42,838 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41633, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T10:28:42,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,839 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] master.ServerManager(517): Registering regionserver=0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,840 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582 2024-11-08T10:28:42,840 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T10:28:42,840 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T10:28:42,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:42,848 DEBUG [RS:2;0e58e25f69ec:36949 {}] zookeeper.ZKUtil(111): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,848 WARN [RS:2;0e58e25f69ec:36949 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T10:28:42,848 INFO [RS:2;0e58e25f69ec:36949 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:42,848 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0e58e25f69ec,36949,1731061722775] 2024-11-08T10:28:42,848 DEBUG [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,857 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T10:28:42,859 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T10:28:42,860 INFO [RS:2;0e58e25f69ec:36949 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T10:28:42,860 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,860 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T10:28:42,860 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T10:28:42,861 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0e58e25f69ec:0, corePoolSize=2, maxPoolSize=2 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,861 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0e58e25f69ec:0, corePoolSize=1, maxPoolSize=1 2024-11-08T10:28:42,862 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:42,862 DEBUG [RS:2;0e58e25f69ec:36949 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0e58e25f69ec:0, corePoolSize=3, maxPoolSize=3 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,862 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,36949,1731061722775-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T10:28:42,877 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T10:28:42,877 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=0e58e25f69ec,36949,1731061722775-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,877 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,878 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.Replication(171): 0e58e25f69ec,36949,1731061722775 started 2024-11-08T10:28:42,892 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T10:28:42,892 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1482): Serving as 0e58e25f69ec,36949,1731061722775, RpcServer on 0e58e25f69ec/172.17.0.2:36949, sessionid=0x1011b4732e60003 2024-11-08T10:28:42,892 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T10:28:42,893 DEBUG [RS:2;0e58e25f69ec:36949 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,893 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,36949,1731061722775' 2024-11-08T10:28:42,893 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T10:28:42,893 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0e58e25f69ec,36949,1731061722775' 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T10:28:42,894 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T10:28:42,895 DEBUG [RS:2;0e58e25f69ec:36949 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T10:28:42,895 INFO [RS:2;0e58e25f69ec:36949 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T10:28:42,895 INFO [RS:2;0e58e25f69ec:36949 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T10:28:42,917 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T10:28:42,919 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56673, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T10:28:42,921 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T10:28:42,921 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T10:28:42,922 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T10:28:42,923 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C40963%2C1731061719230.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,40963,1731061719230, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:42,945 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,40963,1731061719230/0e58e25f69ec%2C40963%2C1731061719230.meta.1731061722923.meta 2024-11-08T10:28:42,946 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:42,946 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:42,946 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T10:28:42,947 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T10:28:42,947 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T10:28:42,947 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T10:28:42,947 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:42,947 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T10:28:42,947 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T10:28:42,948 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T10:28:42,948 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T10:28:42,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:42,950 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2, delay=965 ms 2024-11-08T10:28:42,951 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:42,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:42,952 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T10:28:42,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T10:28:42,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:42,954 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052, delay=958 ms 2024-11-08T10:28:42,956 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:42,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:42,956 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T10:28:42,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T10:28:42,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:42,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:42,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T10:28:42,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T10:28:42,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:42,959 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556, delay=904 ms 2024-11-08T10:28:42,960 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:42,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T10:28:42,961 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T10:28:42,961 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:42,962 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740 2024-11-08T10:28:42,962 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T10:28:42,962 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T10:28:42,963 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T10:28:42,963 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T10:28:42,964 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68260662, jitterRate=0.017163127660751343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T10:28:42,964 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T10:28:42,965 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731061722947Writing region info on filesystem at 1731061722947Initializing all the Stores at 1731061722947Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061722947Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061722948 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061722948Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731061722948Cleaning up temporary data from old regions at 1731061722962 (+14 ms)Running coprocessor post-open hooks at 1731061722964 (+2 ms)Region opened successfully at 1731061722965 (+1 ms) 2024-11-08T10:28:42,966 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731061722916 2024-11-08T10:28:42,968 DEBUG [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T10:28:42,968 INFO [RS_OPEN_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T10:28:42,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:42,970 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0e58e25f69ec,40963,1731061719230, state=OPEN 2024-11-08T10:28:42,991 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42311 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=0e58e25f69ec,40963,1731061719230, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-08T10:28:42,995 INFO [RS:2;0e58e25f69ec:36949 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T10:28:42,996 INFO [RS:2;0e58e25f69ec:36949 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=0e58e25f69ec%2C36949%2C1731061722775, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,36949,1731061722775, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs, maxLogs=55 2024-11-08T10:28:42,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:42,996 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:42,996 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T10:28:42,996 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:42,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T10:28:42,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T10:28:42,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0e58e25f69ec,40963,1731061719230 in 232 msec 2024-11-08T10:28:43,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T10:28:43,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 414 msec 2024-11-08T10:28:43,002 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(207): 0e58e25f69ec,45305,1731061719293 had 2 regions 2024-11-08T10:28:43,003 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 0e58e25f69ec,45305,1731061719293, splitWal=true, meta=true, isMeta: false 2024-11-08T10:28:43,004 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293-splitting dir is empty, no logs to split. 2024-11-08T10:28:43,004 INFO [PEWorker-4 {}] master.SplitWALManager(105): 0e58e25f69ec,45305,1731061719293 WAL count=0, meta=false 2024-11-08T10:28:43,005 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T10:28:43,005 INFO [PEWorker-4 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,45305,1731061719293-splitting dir is empty, no logs to split. 2024-11-08T10:28:43,006 INFO [PEWorker-4 {}] master.SplitWALManager(105): 0e58e25f69ec,45305,1731061719293 WAL count=0, meta=false 2024-11-08T10:28:43,006 DEBUG [PEWorker-4 {}] procedure.ServerCrashProcedure(329): Check if 0e58e25f69ec,45305,1731061719293 WAL splitting is done? wals=0, meta=false 2024-11-08T10:28:43,006 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T10:28:43,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN}] 2024-11-08T10:28:43,008 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:28:43,008 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T10:28:43,009 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T10:28:43,009 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T10:28:43,009 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN 2024-11-08T10:28:43,010 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T10:28:43,017 INFO [RS:2;0e58e25f69ec:36949 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/WALs/0e58e25f69ec,36949,1731061722775/0e58e25f69ec%2C36949%2C1731061722775.1731061722997 2024-11-08T10:28:43,017 DEBUG [RS:2;0e58e25f69ec:36949 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(204): Hosts are {0e58e25f69ec=0} racks are {/default-rack=0} 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T10:28:43,161 INFO [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T10:28:43,161 INFO [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T10:28:43,161 DEBUG [0e58e25f69ec:42311 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T10:28:43,162 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=437c85594b3e91246859c460e224f047, regionState=OPENING, regionLocation=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:43,163 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 0e58e25f69ec:45305 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0e58e25f69ec/172.17.0.2:45305 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T10:28:43,164 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1, error=java.net.ConnectException: Call to address=0e58e25f69ec:45305 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0e58e25f69ec/172.17.0.2:45305 2024-11-08T10:28:43,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T10:28:43,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,45305,1731061719293, seqNum=-1 from cache 2024-11-08T10:28:43,166 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 0e58e25f69ec:45305 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 0e58e25f69ec/172.17.0.2:45305 2024-11-08T10:28:43,276 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T10:28:43,276 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0e58e25f69ec,40963,1731061719230, seqNum=-1] 2024-11-08T10:28:43,276 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T10:28:43,278 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59223, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T10:28:43,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN because future has completed 2024-11-08T10:28:43,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,40963,1731061719230}] 2024-11-08T10:28:43,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T10:28:43,393 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T10:28:43,393 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:43,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,393 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T10:28:43,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T10:28:43,393 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1701651951, stopped=false 2024-11-08T10:28:43,393 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0e58e25f69ec,42311,1731061719089 2024-11-08T10:28:43,434 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,434 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 437c85594b3e91246859c460e224f047, NAME => 'testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.', STARTKEY => '', ENDKEY => ''} 2024-11-08T10:28:43,435 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731061720565 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,435 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T10:28:43,435 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,435 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,436 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,437 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 437c85594b3e91246859c460e224f047 columnFamilyName family 2024-11-08T10:28:43,437 DEBUG [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T10:28:43,438 DEBUG [StoreFileOpener-437c85594b3e91246859c460e224f047-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c, delay=955 ms 2024-11-08T10:28:43,439 DEBUG [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:43,439 INFO [StoreOpener-437c85594b3e91246859c460e224f047-1 {}] regionserver.HStore(327): Store=437c85594b3e91246859c460e224f047/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T10:28:43,440 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,441 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,441 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,441 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,442 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,443 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,443 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 437c85594b3e91246859c460e224f047; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66794948, jitterRate=-0.004677712917327881}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T10:28:43,443 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,444 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 437c85594b3e91246859c460e224f047: Running coprocessor pre-open hook at 1731061723435Writing region info on filesystem at 1731061723435Initializing all the Stores at 1731061723436 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731061723436Cleaning up temporary data from old regions at 1731061723442 (+6 ms)Running coprocessor post-open hooks at 1731061723443 (+1 ms)Region opened successfully at 1731061723444 (+1 ms) 2024-11-08T10:28:43,445 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047., pid=11, masterSystemTime=1731061723432 2024-11-08T10:28:43,447 DEBUG [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,448 INFO [RS_OPEN_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,448 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=437c85594b3e91246859c460e224f047, regionState=OPEN, openSeqNum=10, regionLocation=0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:43,450 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,40963,1731061719230 because future has completed 2024-11-08T10:28:43,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T10:28:43,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 437c85594b3e91246859c460e224f047, server=0e58e25f69ec,40963,1731061719230 in 171 msec 2024-11-08T10:28:43,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-08T10:28:43,454 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 0e58e25f69ec,45305,1731061719293 after splitting done 2024-11-08T10:28:43,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731061720565, region=437c85594b3e91246859c460e224f047, ASSIGN in 445 msec 2024-11-08T10:28:43,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 0e58e25f69ec,45305,1731061719293, splitWal=true, meta=true in 881 msec 2024-11-08T10:28:43,472 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:43,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:43,472 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T10:28:43,472 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:43,472 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:43,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:43,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T10:28:43,472 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T10:28:43,473 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:43,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,473 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0e58e25f69ec,40963,1731061719230' ***** 2024-11-08T10:28:43,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:43,473 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T10:28:43,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:43,473 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0e58e25f69ec,36949,1731061722775' ***** 2024-11-08T10:28:43,473 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T10:28:43,473 INFO [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T10:28:43,473 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T10:28:43,473 INFO [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence 2024-11-08T10:28:43,473 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T10:28:43,474 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T10:28:43,474 INFO [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T10:28:43,474 INFO [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50d85f19-ae7e-7898-4a62-66b9b490953c/bucket.persistence 2024-11-08T10:28:43,474 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T10:28:43,474 INFO [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T10:28:43,475 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T10:28:43,484 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1488): Thread RS:0;0e58e25f69ec:40963 started persisting bucket cache to file 2024-11-08T10:28:43,484 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:43,484 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:43,485 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1488): Thread RS:2;0e58e25f69ec:36949 started persisting bucket cache to file 2024-11-08T10:28:43,485 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T10:28:43,485 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T10:28:43,487 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:43,487 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T10:28:43,487 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061723484 2024-11-08T10:28:43,487 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731061723485 2024-11-08T10:28:43,487 DEBUG [RS:0;0e58e25f69ec:40963 {}] bucket.BucketCache(1510): Thread RS:0;0e58e25f69ec:40963 finished persisting bucket cache to file, renaming 2024-11-08T10:28:43,487 DEBUG [RS:2;0e58e25f69ec:36949 {}] bucket.BucketCache(1510): Thread RS:2;0e58e25f69ec:36949 finished persisting bucket cache to file, renaming 2024-11-08T10:28:43,487 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:28:43,487 INFO [RS:0;0e58e25f69ec:40963 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:28:43,487 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T10:28:43,488 INFO [RS:0;0e58e25f69ec:40963 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T10:28:43,488 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(3091): Received CLOSE for 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T10:28:43,488 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(959): stopping server 0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:43,488 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(959): stopping server 0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;0e58e25f69ec:36949. 2024-11-08T10:28:43,488 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:43,488 DEBUG [RS:2;0e58e25f69ec:36949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:43,488 INFO [RS:0;0e58e25f69ec:40963 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0e58e25f69ec:40963. 2024-11-08T10:28:43,488 DEBUG [RS:2;0e58e25f69ec:36949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,488 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:43,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 437c85594b3e91246859c460e224f047, disabling compactions & flushes 2024-11-08T10:28:43,488 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(976): stopping server 0e58e25f69ec,36949,1731061722775; all regions closed. 2024-11-08T10:28:43,488 DEBUG [RS:0;0e58e25f69ec:40963 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T10:28:43,488 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,488 DEBUG [RS:0;0e58e25f69ec:40963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. after waiting 0 ms 2024-11-08T10:28:43,488 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,489 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:28:43,489 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:28:43,489 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:28:43,489 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T10:28:43,489 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T10:28:43,489 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 437c85594b3e91246859c460e224f047=testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047.} 2024-11-08T10:28:43,489 DEBUG [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 437c85594b3e91246859c460e224f047 2024-11-08T10:28:43,489 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T10:28:43,489 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T10:28:43,489 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T10:28:43,489 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T10:28:43,489 DEBUG [StoreFileCloser-437c85594b3e91246859c460e224f047-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/family/9002343d382740e4a18e8511bcd0ac4c 2024-11-08T10:28:43,489 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T10:28:43,490 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T10:28:43,490 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-08T10:28:43,490 DEBUG [RS:2;0e58e25f69ec:36949 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:43,490 INFO [RS:2;0e58e25f69ec:36949 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C36949%2C1731061722775:(num 1731061722997) 2024-11-08T10:28:43,490 DEBUG [RS:2;0e58e25f69ec:36949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,490 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.ChoreService(370): Chore service for: regionserver/0e58e25f69ec:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:43,491 INFO [RS:2;0e58e25f69ec:36949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36949 2024-11-08T10:28:43,492 INFO [regionserver/0e58e25f69ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:43,494 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/default/testBlockEvictionOnGracefulStop1731061720565/437c85594b3e91246859c460e224f047/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T10:28:43,494 INFO [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,494 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 437c85594b3e91246859c460e224f047: Waiting for close lock at 1731061723488Running coprocessor pre-close hooks at 1731061723488Disabling compacts and flushes for region at 1731061723488Disabling writes for close at 1731061723488Writing region close event to WAL at 1731061723490 (+2 ms)Running coprocessor post-close hooks at 1731061723494 (+4 ms)Closed at 1731061723494 2024-11-08T10:28:43,494 DEBUG [RS_CLOSE_REGION-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047. 2024-11-08T10:28:43,499 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c260321a36b404499cafca0d7e813a9 2024-11-08T10:28:43,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:43,508 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0e58e25f69ec,36949,1731061722775 2024-11-08T10:28:43,508 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:43,512 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/0c260321a36b404499cafca0d7e813a9 is 209, key is testBlockEvictionOnGracefulStop1731061720565,,1731061720573.437c85594b3e91246859c460e224f047./info:regioninfo/1731061723447/Put/seqid=0 2024-11-08T10:28:43,512 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/0c260321a36b404499cafca0d7e813a9 2024-11-08T10:28:43,514 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/.tmp/info/0c260321a36b404499cafca0d7e813a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/0c260321a36b404499cafca0d7e813a9 2024-11-08T10:28:43,515 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/0c260321a36b404499cafca0d7e813a9, delay=966 ms 2024-11-08T10:28:43,516 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/0c260321a36b404499cafca0d7e813a9, entries=8, sequenceid=19, filesize=6.6 K 2024-11-08T10:28:43,517 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false 2024-11-08T10:28:43,518 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/b4229a0d9caa463caf64f2358f6d23f2 2024-11-08T10:28:43,518 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/info/0c260321a36b404499cafca0d7e813a9 2024-11-08T10:28:43,519 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/ns/482791e400b845babc4e9d7ba71cc052 2024-11-08T10:28:43,519 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/table/9b5db6f579534ccd913c48a4e22d8556 2024-11-08T10:28:43,522 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0e58e25f69ec,36949,1731061722775] 2024-11-08T10:28:43,523 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-08T10:28:43,523 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T10:28:43,523 INFO [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:43,524 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731061723489Running coprocessor pre-close hooks at 1731061723489Disabling compacts and flushes for region at 1731061723489Disabling writes for close at 1731061723490 (+1 ms)Obtaining lock to block concurrent updates at 1731061723490Preparing flush snapshotting stores in 1588230740 at 1731061723490Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731061723490Flushing stores of hbase:meta,,1.1588230740 at 1731061723490Flushing 1588230740/info: creating writer at 1731061723490Flushing 1588230740/info: appending metadata at 1731061723511 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731061723511Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d9f73a5: reopening flushed file at 1731061723514 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false at 1731061723517 (+3 ms)Writing region close event to WAL at 1731061723519 (+2 ms)Running coprocessor post-close hooks at 1731061723523 (+4 ms)Closed at 1731061723523 2024-11-08T10:28:43,524 DEBUG [RS_CLOSE_META-regionserver/0e58e25f69ec:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T10:28:43,530 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0e58e25f69ec,36949,1731061722775 already deleted, retry=false 2024-11-08T10:28:43,530 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0e58e25f69ec,36949,1731061722775 expired; onlineServers=1 2024-11-08T10:28:43,550 INFO [regionserver/0e58e25f69ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:43,563 INFO [regionserver/0e58e25f69ec:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:43,622 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,622 INFO [RS:2;0e58e25f69ec:36949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:43,622 DEBUG [pool-298-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36949-0x1011b4732e60003, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,622 INFO [RS:2;0e58e25f69ec:36949 {}] regionserver.HRegionServer(1031): Exiting; stopping=0e58e25f69ec,36949,1731061722775; zookeeper connection closed. 2024-11-08T10:28:43,623 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c1490e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c1490e 2024-11-08T10:28:43,689 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(976): stopping server 0e58e25f69ec,40963,1731061719230; all regions closed. 2024-11-08T10:28:43,691 DEBUG [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:43,691 INFO [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C40963%2C1731061719230.meta:.meta(num 1731061722923) 2024-11-08T10:28:43,693 DEBUG [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/oldWALs 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0e58e25f69ec%2C40963%2C1731061719230:(num 1731061720083) 2024-11-08T10:28:43,693 DEBUG [RS:0;0e58e25f69ec:40963 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.ChoreService(370): Chore service for: regionserver/0e58e25f69ec:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:43,693 INFO [regionserver/0e58e25f69ec:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:43,693 INFO [RS:0;0e58e25f69ec:40963 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40963 2024-11-08T10:28:43,705 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0e58e25f69ec,40963,1731061719230 2024-11-08T10:28:43,705 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T10:28:43,705 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:43,706 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0e58e25f69ec,40963,1731061719230] 2024-11-08T10:28:43,722 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0e58e25f69ec,40963,1731061719230 already deleted, retry=false 2024-11-08T10:28:43,722 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0e58e25f69ec,40963,1731061719230 expired; onlineServers=0 2024-11-08T10:28:43,722 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0e58e25f69ec,42311,1731061719089' ***** 2024-11-08T10:28:43,722 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T10:28:43,722 INFO [M:0;0e58e25f69ec:42311 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T10:28:43,722 INFO [M:0;0e58e25f69ec:42311 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T10:28:43,722 DEBUG [M:0;0e58e25f69ec:42311 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T10:28:43,723 DEBUG [M:0;0e58e25f69ec:42311 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T10:28:43,723 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T10:28:43,723 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061719783 {}] cleaner.HFileCleaner(306): Exit Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.large.0-1731061719783,5,FailOnTimeoutGroup] 2024-11-08T10:28:43,723 DEBUG [master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061719783 {}] cleaner.HFileCleaner(306): Exit Thread[master/0e58e25f69ec:0:becomeActiveMaster-HFileCleaner.small.0-1731061719783,5,FailOnTimeoutGroup] 2024-11-08T10:28:43,723 INFO [M:0;0e58e25f69ec:42311 {}] hbase.ChoreService(370): Chore service for: master/0e58e25f69ec:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T10:28:43,723 INFO [M:0;0e58e25f69ec:42311 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T10:28:43,723 DEBUG [M:0;0e58e25f69ec:42311 {}] master.HMaster(1795): Stopping service threads 2024-11-08T10:28:43,723 INFO [M:0;0e58e25f69ec:42311 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T10:28:43,723 INFO [M:0;0e58e25f69ec:42311 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T10:28:43,723 INFO [M:0;0e58e25f69ec:42311 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T10:28:43,724 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T10:28:43,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T10:28:43,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T10:28:43,730 DEBUG [M:0;0e58e25f69ec:42311 {}] zookeeper.ZKUtil(347): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T10:28:43,730 WARN [M:0;0e58e25f69ec:42311 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T10:28:43,731 INFO [M:0;0e58e25f69ec:42311 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/.lastflushedseqids 2024-11-08T10:28:43,736 INFO [M:0;0e58e25f69ec:42311 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T10:28:43,736 INFO [M:0;0e58e25f69ec:42311 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T10:28:43,736 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T10:28:43,736 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:43,736 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:43,736 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T10:28:43,736 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:43,737 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-08T10:28:43,744 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bbdbd64513d24e96a0059cb10d155580 2024-11-08T10:28:43,756 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbdbd64513d24e96a0059cb10d155580 is 82, key is hbase:meta,,1/info:regioninfo/1731061722968/Put/seqid=0 2024-11-08T10:28:43,757 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbdbd64513d24e96a0059cb10d155580 2024-11-08T10:28:43,763 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 215c067e332046f3ada73603ebaab6b5 2024-11-08T10:28:43,776 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/215c067e332046f3ada73603ebaab6b5 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731061720969/Put/seqid=0 2024-11-08T10:28:43,776 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/215c067e332046f3ada73603ebaab6b5 2024-11-08T10:28:43,782 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6d751a3e80b3462f87a86c7bd17e11a3 2024-11-08T10:28:43,795 DEBUG [M:0;0e58e25f69ec:42311 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d751a3e80b3462f87a86c7bd17e11a3 is 69, key is 0e58e25f69ec,36949,1731061722775/rs:state/1731061722839/Put/seqid=0 2024-11-08T10:28:43,795 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d751a3e80b3462f87a86c7bd17e11a3 2024-11-08T10:28:43,797 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6d751a3e80b3462f87a86c7bd17e11a3 2024-11-08T10:28:43,797 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bbdbd64513d24e96a0059cb10d155580 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbdbd64513d24e96a0059cb10d155580 2024-11-08T10:28:43,799 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bbdbd64513d24e96a0059cb10d155580, entries=14, sequenceid=102, filesize=5.9 K 2024-11-08T10:28:43,799 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/215c067e332046f3ada73603ebaab6b5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/215c067e332046f3ada73603ebaab6b5 2024-11-08T10:28:43,800 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/215c067e332046f3ada73603ebaab6b5, entries=11, sequenceid=102, filesize=6.5 K 2024-11-08T10:28:43,801 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6d751a3e80b3462f87a86c7bd17e11a3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6d751a3e80b3462f87a86c7bd17e11a3 2024-11-08T10:28:43,802 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6d751a3e80b3462f87a86c7bd17e11a3 2024-11-08T10:28:43,802 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/84c8b5c2-0bb4-720f-49a4-40e861ebaa6c/ae9722fe-83af-50cb-5f03-a9e554b84582/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6d751a3e80b3462f87a86c7bd17e11a3, entries=3, sequenceid=102, filesize=5.3 K 2024-11-08T10:28:43,803 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false 2024-11-08T10:28:43,804 INFO [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T10:28:43,805 DEBUG [M:0;0e58e25f69ec:42311 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731061723736Disabling compacts and flushes for region at 1731061723736Disabling writes for close at 1731061723736Obtaining lock to block concurrent updates at 1731061723737 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731061723737Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731061723737Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731061723737Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731061723737Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731061723756 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731061723756Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731061723758 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731061723775 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731061723775Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731061723778 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731061723794 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731061723794Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@473ca5e: reopening flushed file at 1731061723797 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@753e8bc6: reopening flushed file at 1731061723799 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42718161: reopening flushed file at 1731061723800 (+1 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false at 1731061723803 (+3 ms)Writing region close event to WAL at 1731061723804 (+1 ms)Closed at 1731061723804 2024-11-08T10:28:43,806 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T10:28:43,806 INFO [M:0;0e58e25f69ec:42311 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T10:28:43,806 INFO [M:0;0e58e25f69ec:42311 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42311 2024-11-08T10:28:43,806 INFO [M:0;0e58e25f69ec:42311 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T10:28:43,814 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,814 INFO [RS:0;0e58e25f69ec:40963 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:43,814 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40963-0x1011b4732e60001, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,814 INFO [RS:0;0e58e25f69ec:40963 {}] regionserver.HRegionServer(1031): Exiting; stopping=0e58e25f69ec,40963,1731061719230; zookeeper connection closed. 2024-11-08T10:28:43,814 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@288713ec {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@288713ec 2024-11-08T10:28:43,814 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T10:28:43,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,931 INFO [M:0;0e58e25f69ec:42311 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T10:28:43,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42311-0x1011b4732e60000, quorum=127.0.0.1:53173, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T10:28:43,933 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T10:28:43,934 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T10:28:43,934 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T10:28:43,940 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 447) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5349 (was 5462)