2024-11-24 13:55:10,508 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 13:55:10,522 main DEBUG Took 0.011598 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 13:55:10,522 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 13:55:10,522 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 13:55:10,529 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 13:55:10,530 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,539 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 13:55:10,553 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,555 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,556 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,556 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,557 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,558 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,559 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,560 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,561 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,561 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,562 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,563 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,564 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,565 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,565 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,567 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,567 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,568 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,568 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,569 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,569 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,570 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,570 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 13:55:10,571 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,571 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 13:55:10,573 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 13:55:10,575 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 13:55:10,577 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 13:55:10,578 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 13:55:10,580 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 13:55:10,580 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 13:55:10,591 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 13:55:10,595 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 13:55:10,597 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 13:55:10,598 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 13:55:10,598 main DEBUG createAppenders(={Console}) 2024-11-24 13:55:10,599 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-24 13:55:10,600 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-24 13:55:10,600 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-24 13:55:10,601 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 13:55:10,601 main DEBUG OutputStream closed 2024-11-24 13:55:10,601 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 13:55:10,601 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 13:55:10,602 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-24 13:55:10,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 13:55:10,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 13:55:10,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 13:55:10,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 13:55:10,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 13:55:10,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 13:55:10,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 13:55:10,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 13:55:10,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 13:55:10,687 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 13:55:10,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 13:55:10,688 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 13:55:10,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 13:55:10,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 13:55:10,689 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 13:55:10,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 13:55:10,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 13:55:10,690 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 13:55:10,692 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 13:55:10,693 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-24 13:55:10,693 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 13:55:10,694 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-24T13:55:10,709 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-24 13:55:10,713 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 13:55:10,713 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T13:55:10,934 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737 2024-11-24T13:55:10,957 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=8783 2024-11-24T13:55:11,037 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T13:55:11,121 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:11,134 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/cluster_af469665-865d-6589-cfae-f8339385b833, deleteOnExit=true 2024-11-24T13:55:11,244 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/cluster_af469665-865d-6589-cfae-f8339385b833/zookeeper_0, clientPort=64565, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/cluster_af469665-865d-6589-cfae-f8339385b833/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/cluster_af469665-865d-6589-cfae-f8339385b833/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T13:55:11,256 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64565 2024-11-24T13:55:11,257 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:11,348 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be with version=8 2024-11-24T13:55:11,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase-staging 2024-11-24T13:55:11,431 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T13:55:11,652 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:11,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:11,663 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:11,675 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:11,675 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:11,675 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:11,811 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T13:55:11,887 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T13:55:11,925 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T13:55:11,936 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T13:55:11,939 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:11,973 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 94774 (auto-detected) 2024-11-24T13:55:11,974 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T13:55:11,991 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36151 2024-11-24T13:55:12,015 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36151 connecting to ZooKeeper ensemble=127.0.0.1:64565 2024-11-24T13:55:12,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361510x0, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:12,105 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36151-0x1016e6a09bf0000 connected 2024-11-24T13:55:12,183 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,185 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:12,197 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be, hbase.cluster.distributed=false 2024-11-24T13:55:12,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:12,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36151 2024-11-24T13:55:12,228 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36151 2024-11-24T13:55:12,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36151 2024-11-24T13:55:12,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36151 2024-11-24T13:55:12,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36151 2024-11-24T13:55:12,311 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:12,312 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,313 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:12,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,313 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:12,316 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:55:12,319 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:12,320 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38857 2024-11-24T13:55:12,322 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38857 connecting to ZooKeeper ensemble=127.0.0.1:64565 2024-11-24T13:55:12,322 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388570x0, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:12,337 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38857-0x1016e6a09bf0001 connected 2024-11-24T13:55:12,337 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:12,344 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:55:12,359 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.cache 2024-11-24T13:55:12,367 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T13:55:12,369 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T13:55:12,371 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T13:55:12,373 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T13:55:12,374 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T13:55:12,381 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T13:55:12,385 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T13:55:12,386 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence is complete. 2024-11-24T13:55:12,388 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:55:12,392 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:55:12,398 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:12,401 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38857 2024-11-24T13:55:12,402 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38857 2024-11-24T13:55:12,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38857 2024-11-24T13:55:12,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38857 2024-11-24T13:55:12,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38857 2024-11-24T13:55:12,430 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:12,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,431 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:12,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:12,431 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:12,432 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:55:12,432 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:12,433 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35625 2024-11-24T13:55:12,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35625 connecting to ZooKeeper ensemble=127.0.0.1:64565 2024-11-24T13:55:12,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,436 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:356250x0, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:12,471 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:356250x0, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:12,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35625-0x1016e6a09bf0002 connected 2024-11-24T13:55:12,471 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:55:12,472 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.cache 2024-11-24T13:55:12,472 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T13:55:12,473 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T13:55:12,473 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T13:55:12,473 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T13:55:12,473 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T13:55:12,474 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T13:55:12,474 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:55:12,474 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T13:55:12,475 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence is complete. 2024-11-24T13:55:12,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:55:12,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:12,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35625 2024-11-24T13:55:12,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35625 2024-11-24T13:55:12,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35625 2024-11-24T13:55:12,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35625 2024-11-24T13:55:12,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35625 2024-11-24T13:55:12,501 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ee00117f0849:36151 2024-11-24T13:55:12,502 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ee00117f0849,36151,1732456511496 2024-11-24T13:55:12,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,514 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ee00117f0849,36151,1732456511496 2024-11-24T13:55:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,546 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T13:55:12,548 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ee00117f0849,36151,1732456511496 from backup master directory 2024-11-24T13:55:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ee00117f0849,36151,1732456511496 2024-11-24T13:55:12,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:12,554 WARN [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:12,555 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ee00117f0849,36151,1732456511496 2024-11-24T13:55:12,557 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T13:55:12,558 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T13:55:12,567 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase.id] with ID: 59aa9ee6-5409-4355-86c8-632eebc8bd5d 2024-11-24T13:55:12,567 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.tmp/hbase.id 2024-11-24T13:55:12,572 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase.id] 2024-11-24T13:55:12,609 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:12,615 INFO [master/ee00117f0849:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T13:55:12,640 INFO [master/ee00117f0849:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-24T13:55:12,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:12,685 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:55:12,688 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T13:55:12,724 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:55:12,729 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:12,777 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store 2024-11-24T13:55:12,795 INFO [master/ee00117f0849:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T13:55:12,798 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:12,800 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:55:12,800 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:12,800 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:12,802 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:55:12,802 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:12,802 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:12,803 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732456512800Disabling compacts and flushes for region at 1732456512800Disabling writes for close at 1732456512802 (+2 ms)Writing region close event to WAL at 1732456512802Closed at 1732456512802 2024-11-24T13:55:12,803 WARN [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/.initializing 2024-11-24T13:55:12,803 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,36151,1732456511496 2024-11-24T13:55:12,815 INFO [master/ee00117f0849:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:12,827 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ee00117f0849%2C36151%2C1732456511496, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,36151,1732456511496, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/oldWALs, maxLogs=10 2024-11-24T13:55:12,859 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,36151,1732456511496/ee00117f0849%2C36151%2C1732456511496.1732456512831 2024-11-24T13:55:12,860 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:12,861 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:12,861 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:12,864 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,865 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T13:55:12,928 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:12,929 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:12,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T13:55:12,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:12,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:12,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T13:55:12,939 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:12,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:12,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T13:55:12,944 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:12,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:12,945 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,948 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,948 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,951 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,952 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,954 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:12,957 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:12,963 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:12,964 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60684489, jitterRate=-0.09573064744472504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:12,971 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732456512877Initializing all the Stores at 1732456512878 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456512878Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456512879 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456512879Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456512879Cleaning up temporary data from old regions at 1732456512952 (+73 ms)Region opened successfully at 1732456512971 (+19 ms) 2024-11-24T13:55:12,973 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T13:55:13,011 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d4043f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:13,052 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T13:55:13,066 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T13:55:13,066 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T13:55:13,070 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T13:55:13,070 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T13:55:13,077 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T13:55:13,077 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T13:55:13,108 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T13:55:13,120 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T13:55:13,177 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T13:55:13,180 INFO [master/ee00117f0849:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T13:55:13,182 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T13:55:13,194 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T13:55:13,196 INFO [master/ee00117f0849:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T13:55:13,201 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T13:55:13,211 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T13:55:13,213 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T13:55:13,219 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T13:55:13,242 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T13:55:13,252 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T13:55:13,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:13,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:13,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:13,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,282 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ee00117f0849,36151,1732456511496, sessionid=0x1016e6a09bf0000, setting cluster-up flag (Was=false) 2024-11-24T13:55:13,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,336 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T13:55:13,339 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ee00117f0849,36151,1732456511496 2024-11-24T13:55:13,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:13,386 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T13:55:13,389 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ee00117f0849,36151,1732456511496 2024-11-24T13:55:13,396 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T13:55:13,474 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:13,486 INFO [master/ee00117f0849:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T13:55:13,491 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(746): ClusterId : 59aa9ee6-5409-4355-86c8-632eebc8bd5d 2024-11-24T13:55:13,491 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(746): ClusterId : 59aa9ee6-5409-4355-86c8-632eebc8bd5d 2024-11-24T13:55:13,494 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:55:13,494 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:55:13,495 INFO [master/ee00117f0849:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T13:55:13,505 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:55:13,505 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:55:13,505 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:55:13,505 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:55:13,502 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ee00117f0849,36151,1732456511496 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T13:55:13,513 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:55:13,513 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:55:13,513 DEBUG [RS:0;ee00117f0849:38857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@326fdca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:13,513 DEBUG [RS:1;ee00117f0849:35625 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7619e286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:13,514 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:13,514 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:13,515 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:13,515 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:13,515 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ee00117f0849:0, corePoolSize=10, maxPoolSize=10 2024-11-24T13:55:13,515 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,516 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:13,516 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,519 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732456543519 2024-11-24T13:55:13,521 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:13,521 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T13:55:13,521 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T13:55:13,522 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T13:55:13,526 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T13:55:13,527 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T13:55:13,527 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:13,527 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T13:55:13,527 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T13:55:13,527 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T13:55:13,531 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ee00117f0849:38857 2024-11-24T13:55:13,531 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ee00117f0849:35625 2024-11-24T13:55:13,530 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,533 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T13:55:13,534 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T13:55:13,535 INFO [RS:0;ee00117f0849:38857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:55:13,535 INFO [RS:1;ee00117f0849:35625 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:55:13,535 INFO [RS:0;ee00117f0849:38857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:55:13,535 INFO [RS:1;ee00117f0849:35625 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:55:13,535 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:55:13,535 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:55:13,535 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T13:55:13,538 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(2659): reportForDuty to master=ee00117f0849,36151,1732456511496 with port=35625, startcode=1732456512430 2024-11-24T13:55:13,538 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T13:55:13,539 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T13:55:13,540 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(2659): reportForDuty to master=ee00117f0849,36151,1732456511496 with port=38857, startcode=1732456512278 2024-11-24T13:55:13,541 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456513540,5,FailOnTimeoutGroup] 2024-11-24T13:55:13,542 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456513541,5,FailOnTimeoutGroup] 2024-11-24T13:55:13,542 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,542 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T13:55:13,543 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,544 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,548 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T13:55:13,549 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:13,552 DEBUG [RS:1;ee00117f0849:35625 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:55:13,552 DEBUG [RS:0;ee00117f0849:38857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:55:13,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:13,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T13:55:13,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:55:13,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:13,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:13,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T13:55:13,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:55:13,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:13,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:13,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T13:55:13,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:55:13,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:13,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:13,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T13:55:13,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:55:13,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:13,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:13,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:55:13,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:13,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:13,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:55:13,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:55:13,584 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:13,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:55:13,595 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:13,598 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67736534, jitterRate=0.009353011846542358}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:13,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732456513557Initializing all the Stores at 1732456513558 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456513558Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456513558Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456513558Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456513558Cleaning up temporary data from old regions at 1732456513584 (+26 ms)Region opened successfully at 1732456513604 (+20 ms) 2024-11-24T13:55:13,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:55:13,605 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:55:13,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:55:13,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:55:13,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:55:13,607 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:13,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732456513604Disabling compacts and flushes for region at 1732456513604Disabling writes for close at 1732456513605 (+1 ms)Writing region close event to WAL at 1732456513607 (+2 ms)Closed at 1732456513607 2024-11-24T13:55:13,610 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:13,610 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T13:55:13,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T13:55:13,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33885, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:55:13,625 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:55:13,630 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:13,632 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,633 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T13:55:13,634 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36151 {}] master.ServerManager(517): Registering regionserver=ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,644 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36151 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,644 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36151 {}] master.ServerManager(517): Registering regionserver=ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,648 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:13,648 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:13,648 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T13:55:13,648 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T13:55:13,648 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:55:13,648 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:55:13,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:13,708 DEBUG [RS:0;ee00117f0849:38857 {}] zookeeper.ZKUtil(111): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,708 DEBUG [RS:1;ee00117f0849:35625 {}] zookeeper.ZKUtil(111): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,708 WARN [RS:0;ee00117f0849:38857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:13,708 WARN [RS:1;ee00117f0849:35625 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:13,709 INFO [RS:1;ee00117f0849:35625 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:13,709 INFO [RS:0;ee00117f0849:38857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:13,709 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,709 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,711 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ee00117f0849,38857,1732456512278] 2024-11-24T13:55:13,712 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ee00117f0849,35625,1732456512430] 2024-11-24T13:55:13,735 INFO [RS:0;ee00117f0849:38857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:55:13,735 INFO [RS:1;ee00117f0849:35625 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:55:13,753 INFO [RS:1;ee00117f0849:35625 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:55:13,753 INFO [RS:0;ee00117f0849:38857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:55:13,758 INFO [RS:0;ee00117f0849:38857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:55:13,758 INFO [RS:1;ee00117f0849:35625 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:55:13,758 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,758 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,759 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:55:13,759 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:55:13,767 INFO [RS:1;ee00117f0849:35625 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:55:13,769 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,770 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,770 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,770 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,770 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,770 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,771 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,772 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,772 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:13,772 DEBUG [RS:1;ee00117f0849:35625 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:13,775 INFO [RS:0;ee00117f0849:38857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:55:13,775 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,776 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,776 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,776 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,776 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,776 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,777 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,778 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:13,778 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:13,778 DEBUG [RS:0;ee00117f0849:38857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,783 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,35625,1732456512430-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:13,784 WARN [ee00117f0849:36151 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,799 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,38857,1732456512278-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:13,807 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:55:13,808 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,35625,1732456512430-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,809 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,809 INFO [RS:1;ee00117f0849:35625 {}] regionserver.Replication(171): ee00117f0849,35625,1732456512430 started 2024-11-24T13:55:13,822 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:55:13,822 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,38857,1732456512278-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,822 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,822 INFO [RS:0;ee00117f0849:38857 {}] regionserver.Replication(171): ee00117f0849,38857,1732456512278 started 2024-11-24T13:55:13,829 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,830 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1482): Serving as ee00117f0849,35625,1732456512430, RpcServer on ee00117f0849/172.17.0.2:35625, sessionid=0x1016e6a09bf0002 2024-11-24T13:55:13,831 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:55:13,831 DEBUG [RS:1;ee00117f0849:35625 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,831 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,35625,1732456512430' 2024-11-24T13:55:13,831 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:55:13,832 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:55:13,833 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:55:13,833 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:55:13,833 DEBUG [RS:1;ee00117f0849:35625 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ee00117f0849,35625,1732456512430 2024-11-24T13:55:13,833 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,35625,1732456512430' 2024-11-24T13:55:13,833 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:55:13,834 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:55:13,835 DEBUG [RS:1;ee00117f0849:35625 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:55:13,835 INFO [RS:1;ee00117f0849:35625 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:55:13,835 INFO [RS:1;ee00117f0849:35625 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:55:13,845 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:13,845 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1482): Serving as ee00117f0849,38857,1732456512278, RpcServer on ee00117f0849/172.17.0.2:38857, sessionid=0x1016e6a09bf0001 2024-11-24T13:55:13,846 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:55:13,846 DEBUG [RS:0;ee00117f0849:38857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,846 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,38857,1732456512278' 2024-11-24T13:55:13,846 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:55:13,847 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:55:13,847 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:55:13,847 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:55:13,847 DEBUG [RS:0;ee00117f0849:38857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ee00117f0849,38857,1732456512278 2024-11-24T13:55:13,848 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,38857,1732456512278' 2024-11-24T13:55:13,848 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:55:13,848 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:55:13,849 DEBUG [RS:0;ee00117f0849:38857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:55:13,849 INFO [RS:0;ee00117f0849:38857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:55:13,849 INFO [RS:0;ee00117f0849:38857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:55:13,940 INFO [RS:1;ee00117f0849:35625 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:13,942 INFO [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C35625%2C1732456512430, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,35625,1732456512430, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:13,950 INFO [RS:0;ee00117f0849:38857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:13,951 INFO [RS:0;ee00117f0849:38857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C38857%2C1732456512278, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38857,1732456512278, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:13,964 INFO [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,35625,1732456512430/ee00117f0849%2C35625%2C1732456512430.1732456513944 2024-11-24T13:55:13,964 DEBUG [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:13,970 INFO [RS:0;ee00117f0849:38857 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38857,1732456512278/ee00117f0849%2C38857%2C1732456512278.1732456513953 2024-11-24T13:55:13,970 DEBUG [RS:0;ee00117f0849:38857 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:14,036 DEBUG [ee00117f0849:36151 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T13:55:14,049 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(204): Hosts are {ee00117f0849=0} racks are {/default-rack=0} 2024-11-24T13:55:14,052 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:55:14,053 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:55:14,053 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:55:14,053 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:55:14,053 INFO [ee00117f0849:36151 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:55:14,053 INFO [ee00117f0849:36151 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:55:14,053 DEBUG [ee00117f0849:36151 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:55:14,061 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ee00117f0849,35625,1732456512430 2024-11-24T13:55:14,066 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,35625,1732456512430, state=OPENING 2024-11-24T13:55:14,094 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T13:55:14,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:14,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:14,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:14,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,104 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,105 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:14,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,35625,1732456512430}] 2024-11-24T13:55:14,110 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,283 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:55:14,287 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54477, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:55:14,295 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T13:55:14,295 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:14,296 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T13:55:14,297 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C35625%2C1732456512430.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,35625,1732456512430, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:14,315 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,35625,1732456512430/ee00117f0849%2C35625%2C1732456512430.meta.1732456514299.meta 2024-11-24T13:55:14,316 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:14,316 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:14,318 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T13:55:14,320 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T13:55:14,324 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T13:55:14,328 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T13:55:14,329 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:14,329 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T13:55:14,329 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T13:55:14,331 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T13:55:14,332 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:55:14,332 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:14,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:14,332 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T13:55:14,333 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:55:14,333 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:14,333 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:14,334 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T13:55:14,334 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:55:14,335 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:14,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:14,335 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T13:55:14,336 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:55:14,336 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:14,336 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:14,336 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:55:14,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:14,338 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:14,338 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:55:14,338 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:55:14,339 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:14,340 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:55:14,340 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73701081, jitterRate=0.09823168814182281}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:14,340 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T13:55:14,342 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732456514330Writing region info on filesystem at 1732456514330Initializing all the Stores at 1732456514330Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456514330Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456514330Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456514330Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456514331 (+1 ms)Cleaning up temporary data from old regions at 1732456514338 (+7 ms)Running coprocessor post-open hooks at 1732456514341 (+3 ms)Region opened successfully at 1732456514342 (+1 ms) 2024-11-24T13:55:14,348 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732456514272 2024-11-24T13:55:14,356 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T13:55:14,356 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T13:55:14,357 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ee00117f0849,35625,1732456512430 2024-11-24T13:55:14,358 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,35625,1732456512430, state=OPEN 2024-11-24T13:55:14,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:14,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:14,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:14,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,386 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:14,387 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ee00117f0849,35625,1732456512430 2024-11-24T13:55:14,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T13:55:14,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,35625,1732456512430 in 280 msec 2024-11-24T13:55:14,396 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T13:55:14,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 774 msec 2024-11-24T13:55:14,401 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:14,401 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T13:55:14,421 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:55:14,423 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ee00117f0849,35625,1732456512430, seqNum=-1] 2024-11-24T13:55:14,440 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:55:14,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:55:14,463 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0310 sec 2024-11-24T13:55:14,463 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732456514463, completionTime=-1 2024-11-24T13:55:14,465 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T13:55:14,466 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T13:55:14,490 INFO [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T13:55:14,490 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732456574490 2024-11-24T13:55:14,490 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732456634490 2024-11-24T13:55:14,490 INFO [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-24T13:55:14,492 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T13:55:14,498 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,499 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,499 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,500 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ee00117f0849:36151, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,500 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,501 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,507 DEBUG [master/ee00117f0849:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T13:55:14,532 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.976sec 2024-11-24T13:55:14,533 INFO [master/ee00117f0849:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T13:55:14,535 INFO [master/ee00117f0849:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T13:55:14,536 INFO [master/ee00117f0849:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T13:55:14,536 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T13:55:14,536 INFO [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T13:55:14,537 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:14,537 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T13:55:14,538 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T13:55:14,553 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T13:55:14,553 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,36151,1732456511496-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:14,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:14,602 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T13:55:14,602 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T13:55:14,605 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ee00117f0849,36151,-1 for getting cluster id 2024-11-24T13:55:14,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T13:55:14,615 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '59aa9ee6-5409-4355-86c8-632eebc8bd5d' 2024-11-24T13:55:14,617 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T13:55:14,618 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "59aa9ee6-5409-4355-86c8-632eebc8bd5d" 2024-11-24T13:55:14,618 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@86cf8d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:14,618 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ee00117f0849,36151,-1] 2024-11-24T13:55:14,621 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T13:55:14,622 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:14,624 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T13:55:14,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:14,627 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:55:14,634 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ee00117f0849,35625,1732456512430, seqNum=-1] 2024-11-24T13:55:14,634 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:55:14,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:55:14,654 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ee00117f0849,36151,1732456511496 2024-11-24T13:55:14,658 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T13:55:14,662 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ee00117f0849,36151,1732456511496 2024-11-24T13:55:14,664 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@333e904f 2024-11-24T13:55:14,664 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T13:55:14,667 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T13:55:14,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732456514654', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:55:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 2024-11-24T13:55:14,685 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T13:55:14,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732456514654" procId is: 4 2024-11-24T13:55:14,687 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:14,689 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T13:55:14,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:14,703 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 293a62ce83bbb49f064df096ed483831, NAME => 'testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732456514654', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1722): Closing 293a62ce83bbb49f064df096ed483831, disabling compactions & flushes 2024-11-24T13:55:14,710 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. after waiting 0 ms 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:14,710 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:14,710 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732456514654-pool-0 {}] regionserver.HRegion(1676): Region close journal for 293a62ce83bbb49f064df096ed483831: Waiting for close lock at 1732456514710Disabling compacts and flushes for region at 1732456514710Disabling writes for close at 1732456514710Writing region close event to WAL at 1732456514710Closed at 1732456514710 2024-11-24T13:55:14,712 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T13:55:14,717 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732456514712"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732456514712"}]},"ts":"1732456514712"} 2024-11-24T13:55:14,725 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T13:55:14,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T13:55:14,729 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732456514654","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732456514727"}]},"ts":"1732456514727"} 2024-11-24T13:55:14,733 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732456514654, state=ENABLING in hbase:meta 2024-11-24T13:55:14,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ee00117f0849=0} racks are {/default-rack=0} 2024-11-24T13:55:14,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:55:14,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:55:14,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:55:14,734 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:55:14,735 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:55:14,735 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:55:14,735 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:55:14,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, ASSIGN}] 2024-11-24T13:55:14,737 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, ASSIGN 2024-11-24T13:55:14,738 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, ASSIGN; state=OFFLINE, location=ee00117f0849,35625,1732456512430; forceNewPlan=false, retain=false 2024-11-24T13:55:14,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:14,895 INFO [ee00117f0849:36151 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:55:14,897 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=OPENING, regionLocation=ee00117f0849,35625,1732456512430 2024-11-24T13:55:14,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, ASSIGN because future has completed 2024-11-24T13:55:14,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430}] 2024-11-24T13:55:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:15,061 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:15,062 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 293a62ce83bbb49f064df096ed483831, NAME => 'testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:15,062 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732456514654 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,063 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:15,063 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,063 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,064 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,069 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 293a62ce83bbb49f064df096ed483831 columnFamilyName family 2024-11-24T13:55:15,069 DEBUG [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:15,070 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(327): Store=293a62ce83bbb49f064df096ed483831/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:15,070 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,072 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,072 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,072 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,072 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,073 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,079 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:15,080 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 293a62ce83bbb49f064df096ed483831; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61851964, jitterRate=-0.07833391427993774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:55:15,080 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:15,081 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 293a62ce83bbb49f064df096ed483831: Running coprocessor pre-open hook at 1732456515063Writing region info on filesystem at 1732456515063Initializing all the Stores at 1732456515064 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456515064Cleaning up temporary data from old regions at 1732456515072 (+8 ms)Running coprocessor post-open hooks at 1732456515080 (+8 ms)Region opened successfully at 1732456515081 (+1 ms) 2024-11-24T13:55:15,082 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831., pid=6, masterSystemTime=1732456515058 2024-11-24T13:55:15,085 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:15,085 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:15,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=OPEN, openSeqNum=2, regionLocation=ee00117f0849,35625,1732456512430 2024-11-24T13:55:15,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430 because future has completed 2024-11-24T13:55:15,092 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T13:55:15,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430 in 185 msec 2024-11-24T13:55:15,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T13:55:15,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, ASSIGN in 357 msec 2024-11-24T13:55:15,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T13:55:15,096 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732456514654","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732456515096"}]},"ts":"1732456515096"} 2024-11-24T13:55:15,099 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732456514654, state=ENABLED in hbase:meta 2024-11-24T13:55:15,100 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T13:55:15,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732456514654 in 422 msec 2024-11-24T13:55:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:15,322 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732456514654 completed 2024-11-24T13:55:15,322 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732456514654 get assigned. Timeout = 60000ms 2024-11-24T13:55:15,325 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:55:15,333 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732456514654 assigned to meta. Checking AM states. 2024-11-24T13:55:15,333 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:55:15,334 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732456514654 assigned. 2024-11-24T13:55:15,344 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732456514654', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831., hostname=ee00117f0849,35625,1732456512430, seqNum=2] 2024-11-24T13:55:15,355 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 293a62ce83bbb49f064df096ed483831 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T13:55:15,377 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:15,415 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/.tmp/family/3c310d511a7d4aefa271dcb0c92f2c37 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T13:55:15,420 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/.tmp/family/3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:15,455 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/.tmp/family/3c310d511a7d4aefa271dcb0c92f2c37 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:15,457 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, delay=901 ms 2024-11-24T13:55:15,459 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T13:55:15,465 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 293a62ce83bbb49f064df096ed483831 in 110ms, sequenceid=6, compaction requested=false 2024-11-24T13:55:15,466 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732456514654' 2024-11-24T13:55:15,468 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 293a62ce83bbb49f064df096ed483831: 2024-11-24T13:55:16,362 DEBUG [hfile-prefetch-1732456515457 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3c310d511a7d4aefa271dcb0c92f2c37_0. 2024-11-24T13:55:16,368 DEBUG [hfile-prefetch-1732456515457 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 3c310d511a7d4aefa271dcb0c92f2c37_117. 2024-11-24T13:55:16,369 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, with total blocks 2, and data blocks 1 2024-11-24T13:55:16,369 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T13:55:16,370 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T13:55:16,370 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37. Will sleep 100ms and try the verification again. 2024-11-24T13:55:16,470 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, with total blocks 2, and data blocks 1 2024-11-24T13:55:16,470 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-24T13:55:16,470 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2293): found block for file 3c310d511a7d4aefa271dcb0c92f2c37 in the backing map. Acquiring read lock for offset 0 2024-11-24T13:55:16,471 DEBUG [hfile-prefetch-1732456515457 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37 has now been fully cached. 2024-11-24T13:55:16,471 DEBUG [hfile-prefetch-1732456515457 {}] hfile.PrefetchExecutor(121): Prefetch completed for 3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:16,475 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T13:55:16,475 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T13:55:16,476 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:16,476 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:16,476 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T13:55:16,477 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T13:55:16,477 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:16,477 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-24T13:55:16,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=293a62ce83bbb49f064df096ed483831, source=ee00117f0849,35625,1732456512430, destination=ee00117f0849,38857,1732456512278, warming up region on ee00117f0849,38857,1732456512278 2024-11-24T13:55:16,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:55:16,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=293a62ce83bbb49f064df096ed483831, source=ee00117f0849,35625,1732456512430, destination=ee00117f0849,38857,1732456512278, running balancer 2024-11-24T13:55:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE 2024-11-24T13:55:16,482 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE 2024-11-24T13:55:16,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44141, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:55:16,484 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=CLOSING, regionLocation=ee00117f0849,35625,1732456512430 2024-11-24T13:55:16,485 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(7855): Warmup {ENCODED => 293a62ce83bbb49f064df096ed483831, NAME => 'testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:16,486 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,487 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 293a62ce83bbb49f064df096ed483831 columnFamilyName family 2024-11-24T13:55:16,487 DEBUG [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:16,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE because future has completed 2024-11-24T13:55:16,489 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-24T13:55:16,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430}] 2024-11-24T13:55:16,493 DEBUG [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:16,493 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(327): Store=293a62ce83bbb49f064df096ed483831/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:16,493 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:16,493 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456516475 2024-11-24T13:55:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1722): Closing 293a62ce83bbb49f064df096ed483831, disabling compactions & flushes 2024-11-24T13:55:16,493 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T13:55:16,493 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. after waiting 0 ms 2024-11-24T13:55:16,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,495 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38857 {}] regionserver.HRegion(1676): Region close journal for 293a62ce83bbb49f064df096ed483831: Waiting for close lock at 1732456516493Disabling compacts and flushes for region at 1732456516493Disabling writes for close at 1732456516493Writing region close event to WAL at 1732456516495 (+2 ms)Closed at 1732456516495 2024-11-24T13:55:16,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-24T13:55:16,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732456516477 2024-11-24T13:55:16,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T13:55:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36151 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T13:55:16,646 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,647 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-24T13:55:16,647 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 293a62ce83bbb49f064df096ed483831, disabling compactions & flushes 2024-11-24T13:55:16,647 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,647 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,648 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. after waiting 0 ms 2024-11-24T13:55:16,648 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,649 DEBUG [StoreFileCloser-293a62ce83bbb49f064df096ed483831-family-1 {}] bucket.BucketCache(768): Removing file 3c310d511a7d4aefa271dcb0c92f2c37 for region 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,650 DEBUG [StoreFileCloser-293a62ce83bbb49f064df096ed483831-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 3c310d511a7d4aefa271dcb0c92f2c37, starting offset: 0, end offset: 9223372036854775807 2024-11-24T13:55:16,651 DEBUG [StoreFileCloser-293a62ce83bbb49f064df096ed483831-family-1 {}] bucket.BucketCache(856): removed key 3c310d511a7d4aefa271dcb0c92f2c37_0 from back map with offset lock 2094080 in the evict process 2024-11-24T13:55:16,656 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T13:55:16,657 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,658 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 293a62ce83bbb49f064df096ed483831: Waiting for close lock at 1732456516647Running coprocessor pre-close hooks at 1732456516647Disabling compacts and flushes for region at 1732456516647Disabling writes for close at 1732456516648 (+1 ms)Writing region close event to WAL at 1732456516651 (+3 ms)Running coprocessor post-close hooks at 1732456516656 (+5 ms)Closed at 1732456516657 (+1 ms) 2024-11-24T13:55:16,658 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 293a62ce83bbb49f064df096ed483831 move to ee00117f0849,38857,1732456512278 record at close sequenceid=6 2024-11-24T13:55:16,662 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,662 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=CLOSED 2024-11-24T13:55:16,665 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430 because future has completed 2024-11-24T13:55:16,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T13:55:16,669 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,35625,1732456512430 in 178 msec 2024-11-24T13:55:16,670 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE; state=CLOSED, location=ee00117f0849,38857,1732456512278; forceNewPlan=false, retain=false 2024-11-24T13:55:16,693 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T13:55:16,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T13:55:16,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:16,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:16,696 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:16,696 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456516694 2024-11-24T13:55:16,696 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T13:55:16,821 INFO [ee00117f0849:36151 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:55:16,821 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=OPENING, regionLocation=ee00117f0849,38857,1732456512278 2024-11-24T13:55:16,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE because future has completed 2024-11-24T13:55:16,828 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,38857,1732456512278}] 2024-11-24T13:55:16,985 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:16,985 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 293a62ce83bbb49f064df096ed483831, NAME => 'testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:16,986 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732456514654 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,987 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:16,987 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,987 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,989 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,990 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 293a62ce83bbb49f064df096ed483831 columnFamilyName family 2024-11-24T13:55:16,990 DEBUG [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:16,993 DEBUG [StoreFileOpener-293a62ce83bbb49f064df096ed483831-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37, delay=927 ms 2024-11-24T13:55:16,996 DEBUG [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:16,996 INFO [StoreOpener-293a62ce83bbb49f064df096ed483831-1 {}] regionserver.HStore(327): Store=293a62ce83bbb49f064df096ed483831/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:16,996 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,997 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,998 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,998 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:16,998 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:17,002 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:17,002 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 293a62ce83bbb49f064df096ed483831; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64133576, jitterRate=-0.044335246086120605}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:55:17,002 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:17,003 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 293a62ce83bbb49f064df096ed483831: Running coprocessor pre-open hook at 1732456516988Writing region info on filesystem at 1732456516988Initializing all the Stores at 1732456516988Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456516988Cleaning up temporary data from old regions at 1732456516998 (+10 ms)Running coprocessor post-open hooks at 1732456517002 (+4 ms)Region opened successfully at 1732456517003 (+1 ms) 2024-11-24T13:55:17,004 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831., pid=9, masterSystemTime=1732456516980 2024-11-24T13:55:17,006 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,007 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,007 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=293a62ce83bbb49f064df096ed483831, regionState=OPEN, openSeqNum=10, regionLocation=ee00117f0849,38857,1732456512278 2024-11-24T13:55:17,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,38857,1732456512278 because future has completed 2024-11-24T13:55:17,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T13:55:17,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 293a62ce83bbb49f064df096ed483831, server=ee00117f0849,38857,1732456512278 in 183 msec 2024-11-24T13:55:17,016 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732456514654, region=293a62ce83bbb49f064df096ed483831, REOPEN/MOVE in 533 msec 2024-11-24T13:55:17,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T13:55:17,035 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:55:17,036 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:17,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,041 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T13:55:17,041 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T13:55:17,041 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-24T13:55:17,041 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ee00117f0849,36151,1732456511496 2024-11-24T13:55:17,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:17,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:17,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:17,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:55:17,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,090 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:55:17,090 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:17,090 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,092 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ee00117f0849,38857,1732456512278' ***** 2024-11-24T13:55:17,092 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:55:17,092 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ee00117f0849,35625,1732456512430' ***** 2024-11-24T13:55:17,092 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:55:17,092 INFO [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T13:55:17,092 INFO [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T13:55:17,093 INFO [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/bucket.persistence 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T13:55:17,093 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T13:55:17,093 INFO [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T13:55:17,093 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T13:55:17,093 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T13:55:17,098 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:55:17,103 INFO [regionserver/ee00117f0849:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:17,103 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1487): Thread RS:1;ee00117f0849:35625 started persisting bucket cache to file 2024-11-24T13:55:17,103 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:17,104 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:17,104 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1487): Thread RS:0;ee00117f0849:38857 started persisting bucket cache to file 2024-11-24T13:55:17,104 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:17,104 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:17,106 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:17,107 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456517104 2024-11-24T13:55:17,107 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:17,107 DEBUG [RS:0;ee00117f0849:38857 {}] bucket.BucketCache(1509): Thread RS:0;ee00117f0849:38857 finished persisting bucket cache to file, renaming 2024-11-24T13:55:17,107 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456517103 2024-11-24T13:55:17,107 DEBUG [RS:1;ee00117f0849:35625 {}] bucket.BucketCache(1509): Thread RS:1;ee00117f0849:35625 finished persisting bucket cache to file, renaming 2024-11-24T13:55:17,107 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:55:17,107 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:55:17,107 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:55:17,107 INFO [RS:0;ee00117f0849:38857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:55:17,107 INFO [RS:1;ee00117f0849:35625 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:55:17,108 INFO [RS:0;ee00117f0849:38857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:55:17,108 INFO [RS:1;ee00117f0849:35625 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:55:17,108 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(959): stopping server ee00117f0849,35625,1732456512430 2024-11-24T13:55:17,108 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(3091): Received CLOSE for 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:17,108 INFO [RS:1;ee00117f0849:35625 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:17,108 INFO [RS:1;ee00117f0849:35625 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ee00117f0849:35625. 2024-11-24T13:55:17,108 DEBUG [RS:1;ee00117f0849:35625 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:17,108 DEBUG [RS:1;ee00117f0849:35625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,108 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(959): stopping server ee00117f0849,38857,1732456512278 2024-11-24T13:55:17,109 INFO [RS:0;ee00117f0849:38857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:17,109 INFO [RS:1;ee00117f0849:35625 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:55:17,109 INFO [RS:0;ee00117f0849:38857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ee00117f0849:38857. 2024-11-24T13:55:17,109 INFO [RS:1;ee00117f0849:35625 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:55:17,109 INFO [RS:1;ee00117f0849:35625 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:55:17,109 DEBUG [RS:0;ee00117f0849:38857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:17,109 DEBUG [RS:0;ee00117f0849:38857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,109 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T13:55:17,109 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:17,109 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 293a62ce83bbb49f064df096ed483831, disabling compactions & flushes 2024-11-24T13:55:17,110 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T13:55:17,110 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,110 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,110 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1325): Online Regions={293a62ce83bbb49f064df096ed483831=testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831.} 2024-11-24T13:55:17,110 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. after waiting 0 ms 2024-11-24T13:55:17,110 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,110 DEBUG [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1351): Waiting on 293a62ce83bbb49f064df096ed483831 2024-11-24T13:55:17,114 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T13:55:17,114 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T13:55:17,114 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:17,114 DEBUG [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T13:55:17,114 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:55:17,114 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:55:17,114 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:55:17,114 DEBUG [StoreFileCloser-293a62ce83bbb49f064df096ed483831-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/family/3c310d511a7d4aefa271dcb0c92f2c37 2024-11-24T13:55:17,114 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:55:17,114 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:55:17,115 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-24T13:55:17,119 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnRegionMove1732456514654/293a62ce83bbb49f064df096ed483831/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T13:55:17,119 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,119 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 293a62ce83bbb49f064df096ed483831: Waiting for close lock at 1732456517109Running coprocessor pre-close hooks at 1732456517109Disabling compacts and flushes for region at 1732456517109Disabling writes for close at 1732456517110 (+1 ms)Writing region close event to WAL at 1732456517115 (+5 ms)Running coprocessor post-close hooks at 1732456517119 (+4 ms)Closed at 1732456517119 2024-11-24T13:55:17,119 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831. 2024-11-24T13:55:17,123 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ce70b07e6a8f48d1be129829a1ff4e56 2024-11-24T13:55:17,139 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ce70b07e6a8f48d1be129829a1ff4e56 is 205, key is testBlockEvictionOnRegionMove1732456514654,,1732456514669.293a62ce83bbb49f064df096ed483831./info:regioninfo/1732456517007/Put/seqid=0 2024-11-24T13:55:17,140 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ce70b07e6a8f48d1be129829a1ff4e56 2024-11-24T13:55:17,148 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on df94f2e4d79947c68d58fa3d9722d881 2024-11-24T13:55:17,160 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/df94f2e4d79947c68d58fa3d9722d881 is 43, key is default/ns:d/1732456514446/Put/seqid=0 2024-11-24T13:55:17,160 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/df94f2e4d79947c68d58fa3d9722d881 2024-11-24T13:55:17,168 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3f1d8c4908764a44b40da1f6cb887d92 2024-11-24T13:55:17,179 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/3f1d8c4908764a44b40da1f6cb887d92 is 78, key is testBlockEvictionOnRegionMove1732456514654/table:state/1732456515096/Put/seqid=0 2024-11-24T13:55:17,180 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/3f1d8c4908764a44b40da1f6cb887d92 2024-11-24T13:55:17,182 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ce70b07e6a8f48d1be129829a1ff4e56 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ce70b07e6a8f48d1be129829a1ff4e56 2024-11-24T13:55:17,183 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ce70b07e6a8f48d1be129829a1ff4e56, delay=963 ms 2024-11-24T13:55:17,184 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ce70b07e6a8f48d1be129829a1ff4e56, entries=15, sequenceid=15, filesize=7.6 K 2024-11-24T13:55:17,185 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/df94f2e4d79947c68d58fa3d9722d881 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/df94f2e4d79947c68d58fa3d9722d881 2024-11-24T13:55:17,186 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/df94f2e4d79947c68d58fa3d9722d881, delay=909 ms 2024-11-24T13:55:17,187 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/df94f2e4d79947c68d58fa3d9722d881, entries=2, sequenceid=15, filesize=5.0 K 2024-11-24T13:55:17,187 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/3f1d8c4908764a44b40da1f6cb887d92 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/3f1d8c4908764a44b40da1f6cb887d92 2024-11-24T13:55:17,189 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/3f1d8c4908764a44b40da1f6cb887d92, delay=916 ms 2024-11-24T13:55:17,190 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/3f1d8c4908764a44b40da1f6cb887d92, entries=2, sequenceid=15, filesize=5.3 K 2024-11-24T13:55:17,190 INFO [regionserver/ee00117f0849:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:17,191 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=15, compaction requested=false 2024-11-24T13:55:17,191 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T13:55:17,192 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ce70b07e6a8f48d1be129829a1ff4e56 2024-11-24T13:55:17,193 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/df94f2e4d79947c68d58fa3d9722d881 2024-11-24T13:55:17,193 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/3f1d8c4908764a44b40da1f6cb887d92 2024-11-24T13:55:17,198 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-24T13:55:17,198 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:55:17,199 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:17,199 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732456517114Running coprocessor pre-close hooks at 1732456517114Disabling compacts and flushes for region at 1732456517114Disabling writes for close at 1732456517114Obtaining lock to block concurrent updates at 1732456517115 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732456517115Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732456517115Flushing stores of hbase:meta,,1.1588230740 at 1732456517115Flushing 1588230740/info: creating writer at 1732456517116 (+1 ms)Flushing 1588230740/info: appending metadata at 1732456517139 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732456517139Flushing 1588230740/ns: creating writer at 1732456517143 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732456517159 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732456517159Flushing 1588230740/table: creating writer at 1732456517163 (+4 ms)Flushing 1588230740/table: appending metadata at 1732456517179 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732456517179Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@751167c5: reopening flushed file at 1732456517182 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12ea3579: reopening flushed file at 1732456517184 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fcb52bc: reopening flushed file at 1732456517187 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=15, compaction requested=false at 1732456517191 (+4 ms)Writing region close event to WAL at 1732456517194 (+3 ms)Running coprocessor post-close hooks at 1732456517198 (+4 ms)Closed at 1732456517198 2024-11-24T13:55:17,199 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:17,311 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(976): stopping server ee00117f0849,38857,1732456512278; all regions closed. 2024-11-24T13:55:17,314 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(976): stopping server ee00117f0849,35625,1732456512430; all regions closed. 2024-11-24T13:55:17,320 DEBUG [RS:0;ee00117f0849:38857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:17,320 DEBUG [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:17,321 INFO [RS:0;ee00117f0849:38857 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C38857%2C1732456512278:(num 1732456513953) 2024-11-24T13:55:17,321 INFO [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C35625%2C1732456512430.meta:.meta(num 1732456514299) 2024-11-24T13:55:17,321 DEBUG [RS:0;ee00117f0849:38857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,321 INFO [RS:0;ee00117f0849:38857 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:17,321 INFO [RS:0;ee00117f0849:38857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:17,321 INFO [RS:0;ee00117f0849:38857 {}] hbase.ChoreService(370): Chore service for: regionserver/ee00117f0849:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:17,322 INFO [RS:0;ee00117f0849:38857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:55:17,322 INFO [RS:0;ee00117f0849:38857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:55:17,322 INFO [regionserver/ee00117f0849:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:17,322 INFO [RS:0;ee00117f0849:38857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:55:17,322 INFO [RS:0;ee00117f0849:38857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:17,322 INFO [RS:0;ee00117f0849:38857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38857 2024-11-24T13:55:17,323 DEBUG [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:17,323 INFO [RS:1;ee00117f0849:35625 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C35625%2C1732456512430:(num 1732456513944) 2024-11-24T13:55:17,323 DEBUG [RS:1;ee00117f0849:35625 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:17,323 INFO [RS:1;ee00117f0849:35625 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:17,324 INFO [RS:1;ee00117f0849:35625 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:17,324 INFO [RS:1;ee00117f0849:35625 {}] hbase.ChoreService(370): Chore service for: regionserver/ee00117f0849:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:17,324 INFO [RS:1;ee00117f0849:35625 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:17,324 INFO [regionserver/ee00117f0849:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:17,324 INFO [RS:1;ee00117f0849:35625 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35625 2024-11-24T13:55:17,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:17,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ee00117f0849,38857,1732456512278 2024-11-24T13:55:17,352 INFO [RS:0;ee00117f0849:38857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:17,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ee00117f0849,35625,1732456512430 2024-11-24T13:55:17,360 INFO [RS:1;ee00117f0849:35625 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:17,360 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007fce484776e0@54f63292 rejected from java.util.concurrent.ThreadPoolExecutor@11f4a668[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-24T13:55:17,369 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ee00117f0849,38857,1732456512278] 2024-11-24T13:55:17,385 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ee00117f0849,38857,1732456512278 already deleted, retry=false 2024-11-24T13:55:17,385 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ee00117f0849,38857,1732456512278 expired; onlineServers=1 2024-11-24T13:55:17,385 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ee00117f0849,35625,1732456512430] 2024-11-24T13:55:17,393 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ee00117f0849,35625,1732456512430 already deleted, retry=false 2024-11-24T13:55:17,393 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ee00117f0849,35625,1732456512430 expired; onlineServers=0 2024-11-24T13:55:17,394 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ee00117f0849,36151,1732456511496' ***** 2024-11-24T13:55:17,394 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T13:55:17,394 INFO [M:0;ee00117f0849:36151 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:17,394 INFO [M:0;ee00117f0849:36151 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:17,394 DEBUG [M:0;ee00117f0849:36151 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T13:55:17,394 DEBUG [M:0;ee00117f0849:36151 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T13:55:17,394 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T13:55:17,394 DEBUG [master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456513541 {}] cleaner.HFileCleaner(306): Exit Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456513541,5,FailOnTimeoutGroup] 2024-11-24T13:55:17,394 DEBUG [master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456513540 {}] cleaner.HFileCleaner(306): Exit Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456513540,5,FailOnTimeoutGroup] 2024-11-24T13:55:17,395 INFO [M:0;ee00117f0849:36151 {}] hbase.ChoreService(370): Chore service for: master/ee00117f0849:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:17,395 INFO [M:0;ee00117f0849:36151 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:17,395 DEBUG [M:0;ee00117f0849:36151 {}] master.HMaster(1795): Stopping service threads 2024-11-24T13:55:17,395 INFO [M:0;ee00117f0849:36151 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T13:55:17,395 INFO [M:0;ee00117f0849:36151 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:55:17,396 INFO [M:0;ee00117f0849:36151 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T13:55:17,396 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T13:55:17,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:17,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,402 DEBUG [M:0;ee00117f0849:36151 {}] zookeeper.ZKUtil(347): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T13:55:17,402 WARN [M:0;ee00117f0849:36151 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T13:55:17,403 INFO [M:0;ee00117f0849:36151 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.lastflushedseqids 2024-11-24T13:55:17,413 INFO [M:0;ee00117f0849:36151 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T13:55:17,413 INFO [M:0;ee00117f0849:36151 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T13:55:17,414 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:55:17,414 INFO [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:17,414 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:17,414 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:55:17,414 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:17,414 INFO [M:0;ee00117f0849:36151 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-24T13:55:17,422 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1143f32300b48009d045f79cce3e88b 2024-11-24T13:55:17,434 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1143f32300b48009d045f79cce3e88b is 82, key is hbase:meta,,1/info:regioninfo/1732456514357/Put/seqid=0 2024-11-24T13:55:17,434 INFO [M:0;ee00117f0849:36151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1143f32300b48009d045f79cce3e88b 2024-11-24T13:55:17,441 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5e2b4edd8c414fafb1c001c1bf5b5754 2024-11-24T13:55:17,454 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e2b4edd8c414fafb1c001c1bf5b5754 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732456515102/Put/seqid=0 2024-11-24T13:55:17,454 INFO [M:0;ee00117f0849:36151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e2b4edd8c414fafb1c001c1bf5b5754 2024-11-24T13:55:17,461 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 22560e7a783f40f09019e2969ecb9df6 2024-11-24T13:55:17,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38857-0x1016e6a09bf0001, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,469 INFO [RS:0;ee00117f0849:38857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:17,469 INFO [RS:0;ee00117f0849:38857 {}] regionserver.HRegionServer(1031): Exiting; stopping=ee00117f0849,38857,1732456512278; zookeeper connection closed. 2024-11-24T13:55:17,470 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5a6d0e19 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5a6d0e19 2024-11-24T13:55:17,472 DEBUG [M:0;ee00117f0849:36151 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22560e7a783f40f09019e2969ecb9df6 is 69, key is ee00117f0849,35625,1732456512430/rs:state/1732456513645/Put/seqid=0 2024-11-24T13:55:17,473 INFO [M:0;ee00117f0849:36151 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22560e7a783f40f09019e2969ecb9df6 2024-11-24T13:55:17,474 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1143f32300b48009d045f79cce3e88b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1143f32300b48009d045f79cce3e88b 2024-11-24T13:55:17,476 INFO [M:0;ee00117f0849:36151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1143f32300b48009d045f79cce3e88b, entries=8, sequenceid=84, filesize=5.5 K 2024-11-24T13:55:17,476 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5e2b4edd8c414fafb1c001c1bf5b5754 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5e2b4edd8c414fafb1c001c1bf5b5754 2024-11-24T13:55:17,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,477 INFO [RS:1;ee00117f0849:35625 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:17,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35625-0x1016e6a09bf0002, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,477 INFO [RS:1;ee00117f0849:35625 {}] regionserver.HRegionServer(1031): Exiting; stopping=ee00117f0849,35625,1732456512430; zookeeper connection closed. 2024-11-24T13:55:17,477 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@38bd4735 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@38bd4735 2024-11-24T13:55:17,477 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-24T13:55:17,478 INFO [M:0;ee00117f0849:36151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5e2b4edd8c414fafb1c001c1bf5b5754, entries=9, sequenceid=84, filesize=6.5 K 2024-11-24T13:55:17,478 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/22560e7a783f40f09019e2969ecb9df6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/22560e7a783f40f09019e2969ecb9df6 2024-11-24T13:55:17,479 INFO [M:0;ee00117f0849:36151 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/22560e7a783f40f09019e2969ecb9df6, entries=2, sequenceid=84, filesize=5.1 K 2024-11-24T13:55:17,480 INFO [M:0;ee00117f0849:36151 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 66ms, sequenceid=84, compaction requested=false 2024-11-24T13:55:17,482 INFO [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:17,482 DEBUG [M:0;ee00117f0849:36151 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732456517414Disabling compacts and flushes for region at 1732456517414Disabling writes for close at 1732456517414Obtaining lock to block concurrent updates at 1732456517414Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732456517414Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732456517415 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732456517415Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732456517415Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732456517433 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732456517433Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732456517436 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732456517453 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732456517453Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732456517456 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732456517472 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732456517472Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dd33e8: reopening flushed file at 1732456517474 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@798891a2: reopening flushed file at 1732456517476 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7472a485: reopening flushed file at 1732456517478 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 66ms, sequenceid=84, compaction requested=false at 1732456517480 (+2 ms)Writing region close event to WAL at 1732456517482 (+2 ms)Closed at 1732456517482 2024-11-24T13:55:17,484 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:17,484 INFO [M:0;ee00117f0849:36151 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T13:55:17,484 INFO [M:0;ee00117f0849:36151 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36151 2024-11-24T13:55:17,484 INFO [M:0;ee00117f0849:36151 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:17,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,611 INFO [M:0;ee00117f0849:36151 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:17,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36151-0x1016e6a09bf0000, quorum=127.0.0.1:64565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:17,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T13:55:17,637 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T13:55:17,638 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,644 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ee00117f0849:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732456517186 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732456515457 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732456516993 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ee00117f0849:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ee00117f0849:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732456517183 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ee00117f0849:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=217 (was 227), ProcessCount=11 (was 11), AvailableMemoryMB=8402 (was 8783) 2024-11-24T13:55:17,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=217, ProcessCount=11, AvailableMemoryMB=8402 2024-11-24T13:55:17,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/hadoop-log-dir so I do NOT create it in target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426 2024-11-24T13:55:17,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/hadoop-tmp-dir so I do NOT create it in target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426 2024-11-24T13:55:17,650 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,652 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/cluster_3ed3160f-1123-4db5-6fbb-0b8402c0d137, deleteOnExit=true 2024-11-24T13:55:17,677 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/cluster_3ed3160f-1123-4db5-6fbb-0b8402c0d137/zookeeper_0, clientPort=50150, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/cluster_3ed3160f-1123-4db5-6fbb-0b8402c0d137/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/cluster_3ed3160f-1123-4db5-6fbb-0b8402c0d137/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T13:55:17,679 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50150 2024-11-24T13:55:17,697 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be with version=8 2024-11-24T13:55:17,697 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase-staging 2024-11-24T13:55:17,701 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:17,701 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,702 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,702 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:17,702 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,702 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:17,702 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T13:55:17,703 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:17,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33583 2024-11-24T13:55:17,706 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33583 connecting to ZooKeeper ensemble=127.0.0.1:50150 2024-11-24T13:55:17,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335830x0, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:17,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33583-0x1016e6a22f10000 connected 2024-11-24T13:55:17,803 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,804 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,806 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,807 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be, hbase.cluster.distributed=false 2024-11-24T13:55:17,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:17,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33583 2024-11-24T13:55:17,812 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33583 2024-11-24T13:55:17,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33583 2024-11-24T13:55:17,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33583 2024-11-24T13:55:17,813 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33583 2024-11-24T13:55:17,828 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:17,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,828 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:17,829 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38621 2024-11-24T13:55:17,831 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38621 connecting to ZooKeeper ensemble=127.0.0.1:50150 2024-11-24T13:55:17,831 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,831 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386210x0, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:17,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,845 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38621-0x1016e6a22f10001 connected 2024-11-24T13:55:17,845 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:55:17,846 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache 2024-11-24T13:55:17,846 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T13:55:17,846 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T13:55:17,846 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T13:55:17,847 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T13:55:17,847 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T13:55:17,847 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T13:55:17,848 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:55:17,848 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T13:55:17,848 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence is complete. 2024-11-24T13:55:17,849 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:55:17,850 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:17,851 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38621 2024-11-24T13:55:17,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38621 2024-11-24T13:55:17,852 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38621 2024-11-24T13:55:17,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38621 2024-11-24T13:55:17,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38621 2024-11-24T13:55:17,869 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:17,869 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:17,870 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41583 2024-11-24T13:55:17,872 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41583 connecting to ZooKeeper ensemble=127.0.0.1:50150 2024-11-24T13:55:17,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,872 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415830x0, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:17,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:415830x0, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:17,878 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41583-0x1016e6a22f10002 connected 2024-11-24T13:55:17,878 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:55:17,879 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache 2024-11-24T13:55:17,879 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T13:55:17,879 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T13:55:17,879 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T13:55:17,879 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T13:55:17,880 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-24T13:55:17,880 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T13:55:17,880 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence is complete. 2024-11-24T13:55:17,880 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T13:55:17,880 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:55:17,881 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T13:55:17,883 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:17,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41583 2024-11-24T13:55:17,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41583 2024-11-24T13:55:17,886 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41583 2024-11-24T13:55:17,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41583 2024-11-24T13:55:17,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41583 2024-11-24T13:55:17,897 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ee00117f0849:33583 2024-11-24T13:55:17,898 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ee00117f0849,33583,1732456517700 2024-11-24T13:55:17,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,903 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ee00117f0849,33583,1732456517700 2024-11-24T13:55:17,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:17,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:17,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,911 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T13:55:17,912 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ee00117f0849,33583,1732456517700 from backup master directory 2024-11-24T13:55:17,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ee00117f0849,33583,1732456517700 2024-11-24T13:55:17,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T13:55:17,919 WARN [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:17,919 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ee00117f0849,33583,1732456517700 2024-11-24T13:55:17,921 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase.id] with ID: 1ff47e21-5dfc-4dd2-bf4d-075d60034024 2024-11-24T13:55:17,921 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.tmp/hbase.id 2024-11-24T13:55:17,924 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/hbase.id] 2024-11-24T13:55:17,962 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:17,962 INFO [master/ee00117f0849:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T13:55:17,966 INFO [master/ee00117f0849:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 4ms. 2024-11-24T13:55:17,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,977 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:17,982 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:55:17,984 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T13:55:17,984 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:18,006 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store 2024-11-24T13:55:18,015 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:18,016 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:55:18,016 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:18,016 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:18,016 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:55:18,016 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:18,016 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:18,016 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732456518016Disabling compacts and flushes for region at 1732456518016Disabling writes for close at 1732456518016Writing region close event to WAL at 1732456518016Closed at 1732456518016 2024-11-24T13:55:18,017 WARN [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/.initializing 2024-11-24T13:55:18,017 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,33583,1732456517700 2024-11-24T13:55:18,023 INFO [master/ee00117f0849:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:18,027 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ee00117f0849%2C33583%2C1732456517700, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,33583,1732456517700, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/oldWALs, maxLogs=10 2024-11-24T13:55:18,042 INFO [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/WALs/ee00117f0849,33583,1732456517700/ee00117f0849%2C33583%2C1732456517700.1732456518027 2024-11-24T13:55:18,043 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:18,043 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:18,043 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:18,043 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,043 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,044 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T13:55:18,048 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,048 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T13:55:18,052 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:18,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T13:55:18,056 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:18,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T13:55:18,061 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:18,062 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,062 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,063 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,063 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,063 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,063 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:18,064 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T13:55:18,070 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:18,071 INFO [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64021146, jitterRate=-0.0460105836391449}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:18,072 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732456518043Initializing all the Stores at 1732456518044 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518044Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456518044Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456518044Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456518044Cleaning up temporary data from old regions at 1732456518063 (+19 ms)Region opened successfully at 1732456518072 (+9 ms) 2024-11-24T13:55:18,073 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T13:55:18,076 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58da4d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:18,078 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T13:55:18,078 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T13:55:18,078 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T13:55:18,079 INFO [master/ee00117f0849:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T13:55:18,079 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-24T13:55:18,080 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-24T13:55:18,080 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T13:55:18,082 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T13:55:18,084 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T13:55:18,102 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T13:55:18,103 INFO [master/ee00117f0849:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T13:55:18,104 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T13:55:18,110 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T13:55:18,111 INFO [master/ee00117f0849:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T13:55:18,113 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T13:55:18,119 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T13:55:18,120 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T13:55:18,127 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T13:55:18,130 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T13:55:18,135 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,145 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ee00117f0849,33583,1732456517700, sessionid=0x1016e6a22f10000, setting cluster-up flag (Was=false) 2024-11-24T13:55:18,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,186 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T13:55:18,188 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ee00117f0849,33583,1732456517700 2024-11-24T13:55:18,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,227 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T13:55:18,229 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ee00117f0849,33583,1732456517700 2024-11-24T13:55:18,229 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T13:55:18,231 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:18,231 INFO [master/ee00117f0849:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T13:55:18,231 INFO [master/ee00117f0849:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T13:55:18,231 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ee00117f0849,33583,1732456517700 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ee00117f0849:0, corePoolSize=5, maxPoolSize=5 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ee00117f0849:0, corePoolSize=10, maxPoolSize=10 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,233 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:18,234 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732456548235 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T13:55:18,235 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T13:55:18,235 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T13:55:18,235 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T13:55:18,236 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,236 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T13:55:18,237 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,238 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T13:55:18,238 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T13:55:18,238 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T13:55:18,239 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456518239,5,FailOnTimeoutGroup] 2024-11-24T13:55:18,239 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456518239,5,FailOnTimeoutGroup] 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,239 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,252 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T13:55:18,252 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:18,257 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:18,258 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T13:55:18,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:55:18,261 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T13:55:18,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:55:18,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,264 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T13:55:18,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:55:18,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,267 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T13:55:18,270 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:55:18,270 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,270 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,270 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:55:18,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:18,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:18,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:55:18,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:55:18,271 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:18,272 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:55:18,276 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:18,276 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72364047, jitterRate=0.07830832898616791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:18,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732456518257Initializing all the Stores at 1732456518257Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518258 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518258Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456518258Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518258Cleaning up temporary data from old regions at 1732456518271 (+13 ms)Region opened successfully at 1732456518277 (+6 ms) 2024-11-24T13:55:18,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:55:18,277 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:55:18,277 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:55:18,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:55:18,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:55:18,278 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:18,278 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732456518277Disabling compacts and flushes for region at 1732456518277Disabling writes for close at 1732456518278 (+1 ms)Writing region close event to WAL at 1732456518278Closed at 1732456518278 2024-11-24T13:55:18,280 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:18,280 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T13:55:18,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T13:55:18,281 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:18,282 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T13:55:18,290 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(746): ClusterId : 1ff47e21-5dfc-4dd2-bf4d-075d60034024 2024-11-24T13:55:18,290 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(746): ClusterId : 1ff47e21-5dfc-4dd2-bf4d-075d60034024 2024-11-24T13:55:18,290 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:55:18,290 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:55:18,320 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:55:18,320 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:55:18,320 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:55:18,320 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:55:18,328 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:55:18,328 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:55:18,329 DEBUG [RS:0;ee00117f0849:38621 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78eb0515, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:18,329 DEBUG [RS:1;ee00117f0849:41583 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549f034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:18,343 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ee00117f0849:41583 2024-11-24T13:55:18,343 INFO [RS:1;ee00117f0849:41583 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:55:18,343 INFO [RS:1;ee00117f0849:41583 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:55:18,343 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:55:18,344 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(2659): reportForDuty to master=ee00117f0849,33583,1732456517700 with port=41583, startcode=1732456517869 2024-11-24T13:55:18,345 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ee00117f0849:38621 2024-11-24T13:55:18,345 DEBUG [RS:1;ee00117f0849:41583 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:55:18,345 INFO [RS:0;ee00117f0849:38621 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:55:18,345 INFO [RS:0;ee00117f0849:38621 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:55:18,345 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:55:18,346 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(2659): reportForDuty to master=ee00117f0849,33583,1732456517700 with port=38621, startcode=1732456517828 2024-11-24T13:55:18,346 DEBUG [RS:0;ee00117f0849:38621 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:55:18,347 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43613, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:55:18,348 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47517, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:55:18,348 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,348 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(517): Registering regionserver=ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,349 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,349 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(517): Registering regionserver=ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,349 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:18,350 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T13:55:18,350 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:55:18,351 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:18,351 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T13:55:18,351 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:55:18,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:18,393 DEBUG [RS:1;ee00117f0849:41583 {}] zookeeper.ZKUtil(111): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,393 WARN [RS:1;ee00117f0849:41583 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:18,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ee00117f0849,38621,1732456517828] 2024-11-24T13:55:18,393 INFO [RS:1;ee00117f0849:41583 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:18,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ee00117f0849,41583,1732456517869] 2024-11-24T13:55:18,394 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,394 DEBUG [RS:0;ee00117f0849:38621 {}] zookeeper.ZKUtil(111): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,394 WARN [RS:0;ee00117f0849:38621 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:18,394 INFO [RS:0;ee00117f0849:38621 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:18,395 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,402 INFO [RS:0;ee00117f0849:38621 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:55:18,402 INFO [RS:1;ee00117f0849:41583 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:55:18,405 INFO [RS:0;ee00117f0849:38621 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:55:18,405 INFO [RS:0;ee00117f0849:38621 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:55:18,405 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,406 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:55:18,406 INFO [RS:0;ee00117f0849:38621 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:55:18,407 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:18,407 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:18,408 DEBUG [RS:0;ee00117f0849:38621 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:18,408 INFO [RS:1;ee00117f0849:41583 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:55:18,409 INFO [RS:1;ee00117f0849:41583 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:55:18,409 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,409 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:55:18,410 INFO [RS:1;ee00117f0849:41583 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:55:18,410 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,410 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,410 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,410 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,410 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,410 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,410 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,410 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,410 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:18,411 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,38621,1732456517828-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:18,411 DEBUG [RS:1;ee00117f0849:41583 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,412 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,41583,1732456517869-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:18,428 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:55:18,429 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,41583,1732456517869-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,429 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,429 INFO [RS:1;ee00117f0849:41583 {}] regionserver.Replication(171): ee00117f0849,41583,1732456517869 started 2024-11-24T13:55:18,432 WARN [ee00117f0849:33583 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T13:55:18,432 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:55:18,432 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,38621,1732456517828-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,433 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,433 INFO [RS:0;ee00117f0849:38621 {}] regionserver.Replication(171): ee00117f0849,38621,1732456517828 started 2024-11-24T13:55:18,443 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,444 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1482): Serving as ee00117f0849,41583,1732456517869, RpcServer on ee00117f0849/172.17.0.2:41583, sessionid=0x1016e6a22f10002 2024-11-24T13:55:18,444 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:55:18,444 DEBUG [RS:1;ee00117f0849:41583 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,444 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,41583,1732456517869' 2024-11-24T13:55:18,444 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ee00117f0849,41583,1732456517869 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,41583,1732456517869' 2024-11-24T13:55:18,445 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:55:18,446 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:55:18,446 DEBUG [RS:1;ee00117f0849:41583 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:55:18,446 INFO [RS:1;ee00117f0849:41583 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:55:18,446 INFO [RS:1;ee00117f0849:41583 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:55:18,453 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,453 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1482): Serving as ee00117f0849,38621,1732456517828, RpcServer on ee00117f0849/172.17.0.2:38621, sessionid=0x1016e6a22f10001 2024-11-24T13:55:18,453 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:55:18,453 DEBUG [RS:0;ee00117f0849:38621 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,453 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,38621,1732456517828' 2024-11-24T13:55:18,453 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:55:18,454 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,38621,1732456517828' 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:55:18,455 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:55:18,456 DEBUG [RS:0;ee00117f0849:38621 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:55:18,456 INFO [RS:0;ee00117f0849:38621 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:55:18,456 INFO [RS:0;ee00117f0849:38621 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:55:18,547 INFO [RS:1;ee00117f0849:41583 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:18,548 INFO [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C41583%2C1732456517869, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,41583,1732456517869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:18,557 INFO [RS:0;ee00117f0849:38621 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:18,558 INFO [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C38621%2C1732456517828, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:18,570 INFO [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,41583,1732456517869/ee00117f0849%2C41583%2C1732456517869.1732456518552 2024-11-24T13:55:18,570 DEBUG [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:18,576 INFO [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828/ee00117f0849%2C38621%2C1732456517828.1732456518559 2024-11-24T13:55:18,577 DEBUG [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:18,682 DEBUG [ee00117f0849:33583 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-24T13:55:18,683 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(204): Hosts are {ee00117f0849=0} racks are {/default-rack=0} 2024-11-24T13:55:18,684 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:55:18,684 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:55:18,685 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:55:18,685 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:55:18,685 INFO [ee00117f0849:33583 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:55:18,685 INFO [ee00117f0849:33583 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:55:18,685 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:55:18,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,687 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,38621,1732456517828, state=OPENING 2024-11-24T13:55:18,710 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T13:55:18,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:18,720 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:18,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,38621,1732456517828}] 2024-11-24T13:55:18,720 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,876 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:55:18,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41145, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:55:18,885 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T13:55:18,885 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:18,885 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T13:55:18,887 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C38621%2C1732456517828.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:18,906 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828/ee00117f0849%2C38621%2C1732456517828.meta.1732456518887.meta 2024-11-24T13:55:18,906 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:18,907 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:18,907 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T13:55:18,907 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T13:55:18,907 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T13:55:18,908 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T13:55:18,908 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:18,908 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T13:55:18,908 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T13:55:18,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T13:55:18,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:55:18,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T13:55:18,910 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:55:18,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T13:55:18,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:55:18,911 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T13:55:18,912 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:55:18,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:18,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:18,913 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:55:18,913 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:18,914 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:18,914 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:55:18,914 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:55:18,915 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:18,915 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:55:18,916 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61841670, jitterRate=-0.07848730683326721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:18,916 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T13:55:18,917 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732456518908Writing region info on filesystem at 1732456518908Initializing all the Stores at 1732456518908Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518909 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518909Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456518909Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456518909Cleaning up temporary data from old regions at 1732456518914 (+5 ms)Running coprocessor post-open hooks at 1732456518916 (+2 ms)Region opened successfully at 1732456518916 2024-11-24T13:55:18,917 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732456518875 2024-11-24T13:55:18,920 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T13:55:18,920 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T13:55:18,920 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,921 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,38621,1732456517828, state=OPEN 2024-11-24T13:55:18,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:18,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:18,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:18,952 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ee00117f0849,38621,1732456517828 2024-11-24T13:55:18,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,952 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:18,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T13:55:18,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,38621,1732456517828 in 232 msec 2024-11-24T13:55:18,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T13:55:18,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 674 msec 2024-11-24T13:55:18,957 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T13:55:18,957 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T13:55:18,960 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:55:18,960 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1] 2024-11-24T13:55:18,961 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:55:18,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39881, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:55:18,968 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 738 msec 2024-11-24T13:55:18,968 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732456518968, completionTime=-1 2024-11-24T13:55:18,968 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-24T13:55:18,968 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732456578971 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732456638971 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,971 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,972 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,972 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ee00117f0849:33583, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,972 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,972 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,975 DEBUG [master/ee00117f0849:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T13:55:18,977 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.058sec 2024-11-24T13:55:18,977 INFO [master/ee00117f0849:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:18,978 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T13:55:18,979 DEBUG [master/ee00117f0849:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T13:55:18,979 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T13:55:18,979 INFO [master/ee00117f0849:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,33583,1732456517700-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:18,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:18,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ee00117f0849,33583,-1 for getting cluster id 2024-11-24T13:55:18,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T13:55:18,992 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1ff47e21-5dfc-4dd2-bf4d-075d60034024' 2024-11-24T13:55:18,992 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T13:55:18,992 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1ff47e21-5dfc-4dd2-bf4d-075d60034024" 2024-11-24T13:55:18,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c8556c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:18,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ee00117f0849,33583,-1] 2024-11-24T13:55:18,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T13:55:18,993 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:18,995 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T13:55:18,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T13:55:18,996 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:55:18,998 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1] 2024-11-24T13:55:18,999 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:55:19,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:55:19,003 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ee00117f0849,33583,1732456517700 2024-11-24T13:55:19,003 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T13:55:19,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ee00117f0849,33583,1732456517700 2024-11-24T13:55:19,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61e6f31d 2024-11-24T13:55:19,004 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T13:55:19,006 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T13:55:19,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732456519003', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T13:55:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 2024-11-24T13:55:19,009 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T13:55:19,009 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:19,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732456519003" procId is: 4 2024-11-24T13:55:19,010 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T13:55:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:19,021 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7c8d18c3271e4f8f2397747fde461fca, NAME => 'testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732456519003', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1722): Closing 7c8d18c3271e4f8f2397747fde461fca, disabling compactions & flushes 2024-11-24T13:55:19,027 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. after waiting 0 ms 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,027 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,027 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732456519003-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7c8d18c3271e4f8f2397747fde461fca: Waiting for close lock at 1732456519027Disabling compacts and flushes for region at 1732456519027Disabling writes for close at 1732456519027Writing region close event to WAL at 1732456519027Closed at 1732456519027 2024-11-24T13:55:19,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T13:55:19,029 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732456519029"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732456519029"}]},"ts":"1732456519029"} 2024-11-24T13:55:19,032 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T13:55:19,033 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T13:55:19,033 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732456519003","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732456519033"}]},"ts":"1732456519033"} 2024-11-24T13:55:19,036 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732456519003, state=ENABLING in hbase:meta 2024-11-24T13:55:19,036 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ee00117f0849=0} racks are {/default-rack=0} 2024-11-24T13:55:19,036 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:55:19,036 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:55:19,037 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:55:19,037 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:55:19,037 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:55:19,037 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:55:19,037 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:55:19,037 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN}] 2024-11-24T13:55:19,038 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN 2024-11-24T13:55:19,039 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN; state=OFFLINE, location=ee00117f0849,38621,1732456517828; forceNewPlan=false, retain=false 2024-11-24T13:55:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:19,190 INFO [ee00117f0849:33583 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T13:55:19,191 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c8d18c3271e4f8f2397747fde461fca, regionState=OPENING, regionLocation=ee00117f0849,38621,1732456517828 2024-11-24T13:55:19,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN because future has completed 2024-11-24T13:55:19,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,38621,1732456517828}] 2024-11-24T13:55:19,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:19,355 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,356 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c8d18c3271e4f8f2397747fde461fca, NAME => 'testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:19,356 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732456519003 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,356 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:19,356 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,356 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,357 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,361 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8d18c3271e4f8f2397747fde461fca columnFamilyName family 2024-11-24T13:55:19,361 DEBUG [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:19,361 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] regionserver.HStore(327): Store=7c8d18c3271e4f8f2397747fde461fca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:19,361 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,362 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,362 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,362 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,362 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,363 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,371 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T13:55:19,371 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 7c8d18c3271e4f8f2397747fde461fca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73946970, jitterRate=0.10189571976661682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:55:19,372 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:19,373 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 7c8d18c3271e4f8f2397747fde461fca: Running coprocessor pre-open hook at 1732456519356Writing region info on filesystem at 1732456519356Initializing all the Stores at 1732456519357 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456519357Cleaning up temporary data from old regions at 1732456519362 (+5 ms)Running coprocessor post-open hooks at 1732456519372 (+10 ms)Region opened successfully at 1732456519373 (+1 ms) 2024-11-24T13:55:19,373 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca., pid=6, masterSystemTime=1732456519353 2024-11-24T13:55:19,376 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,377 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:19,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=7c8d18c3271e4f8f2397747fde461fca, regionState=OPEN, openSeqNum=2, regionLocation=ee00117f0849,38621,1732456517828 2024-11-24T13:55:19,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,38621,1732456517828 because future has completed 2024-11-24T13:55:19,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T13:55:19,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,38621,1732456517828 in 182 msec 2024-11-24T13:55:19,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T13:55:19,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN in 347 msec 2024-11-24T13:55:19,387 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T13:55:19,388 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732456519003","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732456519387"}]},"ts":"1732456519387"} 2024-11-24T13:55:19,390 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732456519003, state=ENABLED in hbase:meta 2024-11-24T13:55:19,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T13:55:19,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732456519003 in 384 msec 2024-11-24T13:55:19,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33583 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T13:55:19,643 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732456519003 completed 2024-11-24T13:55:19,643 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732456519003 get assigned. Timeout = 60000ms 2024-11-24T13:55:19,643 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:55:19,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732456519003 assigned to meta. Checking AM states. 2024-11-24T13:55:19,651 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T13:55:19,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732456519003 assigned. 2024-11-24T13:55:19,656 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732456519003', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca., hostname=ee00117f0849,38621,1732456517828, seqNum=2] 2024-11-24T13:55:19,661 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7c8d18c3271e4f8f2397747fde461fca 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-24T13:55:19,670 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:19,681 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/.tmp/family/2c2b266a83fb4c2ebf3a6dfcc2db960d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-24T13:55:19,682 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/.tmp/family/2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:19,684 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/.tmp/family/2c2b266a83fb4c2ebf3a6dfcc2db960d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:19,686 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, delay=924 ms 2024-11-24T13:55:19,687 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-24T13:55:19,687 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 7c8d18c3271e4f8f2397747fde461fca in 26ms, sequenceid=6, compaction requested=false 2024-11-24T13:55:19,687 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732456519003' 2024-11-24T13:55:19,688 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7c8d18c3271e4f8f2397747fde461fca: 2024-11-24T13:55:19,831 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T13:55:20,611 DEBUG [hfile-prefetch-1732456516993 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2c2b266a83fb4c2ebf3a6dfcc2db960d_0. 2024-11-24T13:55:20,612 DEBUG [hfile-prefetch-1732456516993 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 2c2b266a83fb4c2ebf3a6dfcc2db960d_117. 2024-11-24T13:55:20,613 DEBUG [hfile-prefetch-1732456516993 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, with total blocks 2, and data blocks 1 2024-11-24T13:55:20,613 DEBUG [hfile-prefetch-1732456516993 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-24T13:55:20,613 DEBUG [hfile-prefetch-1732456516993 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-24T13:55:20,613 DEBUG [hfile-prefetch-1732456516993 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d. Will sleep 100ms and try the verification again. 2024-11-24T13:55:20,689 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;ee00117f0849:38621,5,FailOnTimeoutGroup] 2024-11-24T13:55:20,689 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ee00117f0849,38621,1732456517828' ***** 2024-11-24T13:55:20,689 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-24T13:55:20,690 INFO [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T13:55:20,690 INFO [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence 2024-11-24T13:55:20,690 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T13:55:20,690 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T13:55:20,690 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T13:55:20,691 INFO [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T13:55:20,691 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T13:55:20,701 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1487): Thread RS:0;ee00117f0849:38621 started persisting bucket cache to file 2024-11-24T13:55:20,701 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:20,701 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:20,704 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-24T13:55:20,704 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732456520701 2024-11-24T13:55:20,704 DEBUG [RS:0;ee00117f0849:38621 {}] bucket.BucketCache(1509): Thread RS:0;ee00117f0849:38621 finished persisting bucket cache to file, renaming 2024-11-24T13:55:20,704 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:55:20,705 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(959): stopping server ee00117f0849,38621,1732456517828 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:20,705 INFO [RS:0;ee00117f0849:38621 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ee00117f0849:38621. 2024-11-24T13:55:20,705 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:20,705 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c8d18c3271e4f8f2397747fde461fca, disabling compactions & flushes 2024-11-24T13:55:20,705 DEBUG [RS:0;ee00117f0849:38621 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:20,705 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:20,705 DEBUG [RS:0;ee00117f0849:38621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:20,705 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. after waiting 0 ms 2024-11-24T13:55:20,706 INFO [RS:0;ee00117f0849:38621 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:55:20,706 INFO [RS:0;ee00117f0849:38621 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:20,706 INFO [RS:0;ee00117f0849:38621 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:55:20,706 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T13:55:20,706 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T13:55:20,706 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7c8d18c3271e4f8f2397747fde461fca=testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.} 2024-11-24T13:55:20,706 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:20,706 DEBUG [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:55:20,706 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:55:20,706 DEBUG [StoreFileCloser-7c8d18c3271e4f8f2397747fde461fca-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:55:20,706 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:55:20,707 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-24T13:55:20,708 WARN [hfile-prefetch-1732456516993 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-24T13:55:20,709 DEBUG [hfile-prefetch-1732456516993 {}] hfile.PrefetchExecutor(121): Prefetch completed for 2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:20,711 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-24T13:55:20,711 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:20,711 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c8d18c3271e4f8f2397747fde461fca: Waiting for close lock at 1732456520705Running coprocessor pre-close hooks at 1732456520705Disabling compacts and flushes for region at 1732456520705Disabling writes for close at 1732456520706 (+1 ms)Writing region close event to WAL at 1732456520707 (+1 ms)Running coprocessor post-close hooks at 1732456520711 (+4 ms)Closed at 1732456520711 2024-11-24T13:55:20,711 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:20,713 INFO [regionserver/ee00117f0849:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:20,715 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:20,729 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ad5a15bc267543759a146c3ec8b4a975 is 209, key is testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca./info:regioninfo/1732456519377/Put/seqid=0 2024-11-24T13:55:20,730 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:20,737 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:20,750 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/c7cac323b3f44120a34e7675b5df570e is 43, key is default/ns:d/1732456518963/Put/seqid=0 2024-11-24T13:55:20,751 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:20,758 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:20,772 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 is 80, key is testBlockEvictionOnGracefulStop1732456519003/table:state/1732456519387/Put/seqid=0 2024-11-24T13:55:20,772 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:20,775 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ad5a15bc267543759a146c3ec8b4a975 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:20,777 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975, delay=938 ms 2024-11-24T13:55:20,778 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975, entries=10, sequenceid=11, filesize=7.0 K 2024-11-24T13:55:20,779 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/ns/c7cac323b3f44120a34e7675b5df570e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:20,779 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e, delay=946 ms 2024-11-24T13:55:20,780 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e, entries=2, sequenceid=11, filesize=5.0 K 2024-11-24T13:55:20,781 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:20,782 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-24T13:55:20,782 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-24T13:55:20,782 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4, delay=990 ms 2024-11-24T13:55:20,782 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:20,782 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:20,784 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4, entries=2, sequenceid=11, filesize=5.3 K 2024-11-24T13:55:20,785 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false 2024-11-24T13:55:20,785 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:20,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:20,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456520782 2024-11-24T13:55:20,785 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-24T13:55:20,786 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:20,786 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:20,790 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-24T13:55:20,790 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:55:20,790 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:20,791 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732456520706Running coprocessor pre-close hooks at 1732456520706Disabling compacts and flushes for region at 1732456520706Disabling writes for close at 1732456520706Obtaining lock to block concurrent updates at 1732456520707 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732456520707Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732456520707Flushing stores of hbase:meta,,1.1588230740 at 1732456520707Flushing 1588230740/info: creating writer at 1732456520708 (+1 ms)Flushing 1588230740/info: appending metadata at 1732456520729 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732456520729Flushing 1588230740/ns: creating writer at 1732456520732 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732456520750 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732456520750Flushing 1588230740/table: creating writer at 1732456520753 (+3 ms)Flushing 1588230740/table: appending metadata at 1732456520771 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732456520771Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21878454: reopening flushed file at 1732456520775 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@590d98b: reopening flushed file at 1732456520778 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a720269: reopening flushed file at 1732456520781 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=11, compaction requested=false at 1732456520785 (+4 ms)Writing region close event to WAL at 1732456520787 (+2 ms)Running coprocessor post-close hooks at 1732456520790 (+3 ms)Closed at 1732456520790 2024-11-24T13:55:20,791 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:20,906 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(976): stopping server ee00117f0849,38621,1732456517828; all regions closed. 2024-11-24T13:55:20,912 DEBUG [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:20,912 INFO [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C38621%2C1732456517828.meta:.meta(num 1732456518887) 2024-11-24T13:55:20,915 DEBUG [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:20,915 INFO [RS:0;ee00117f0849:38621 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C38621%2C1732456517828:(num 1732456518559) 2024-11-24T13:55:20,915 DEBUG [RS:0;ee00117f0849:38621 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:20,915 INFO [RS:0;ee00117f0849:38621 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:20,915 INFO [RS:0;ee00117f0849:38621 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:20,916 INFO [RS:0;ee00117f0849:38621 {}] hbase.ChoreService(370): Chore service for: regionserver/ee00117f0849:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:20,916 INFO [RS:0;ee00117f0849:38621 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:20,916 INFO [regionserver/ee00117f0849:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:20,916 INFO [RS:0;ee00117f0849:38621 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38621 2024-11-24T13:55:20,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:20,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ee00117f0849,38621,1732456517828 2024-11-24T13:55:20,952 INFO [RS:0;ee00117f0849:38621 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:20,953 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ee00117f0849,38621,1732456517828] 2024-11-24T13:55:20,968 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ee00117f0849,38621,1732456517828 already deleted, retry=false 2024-11-24T13:55:20,968 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ee00117f0849,38621,1732456517828 on ee00117f0849,33583,1732456517700 2024-11-24T13:55:20,973 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ee00117f0849,38621,1732456517828, splitWal=true, meta=true 2024-11-24T13:55:20,975 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for ee00117f0849,38621,1732456517828 (carryingMeta=true) ee00117f0849,38621,1732456517828/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1b75aa4c[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T13:55:20,976 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ee00117f0849,38621,1732456517828, splitWal=true, meta=true 2024-11-24T13:55:20,977 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ee00117f0849,38621,1732456517828, splitWal=true, meta=true, isMeta: true 2024-11-24T13:55:20,978 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828-splitting 2024-11-24T13:55:20,978 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828-splitting dir is empty, no logs to split. 2024-11-24T13:55:20,978 INFO [PEWorker-2 {}] master.SplitWALManager(105): ee00117f0849,38621,1732456517828 WAL count=0, meta=true 2024-11-24T13:55:20,980 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828-splitting dir is empty, no logs to split. 2024-11-24T13:55:20,980 INFO [PEWorker-2 {}] master.SplitWALManager(105): ee00117f0849,38621,1732456517828 WAL count=0, meta=true 2024-11-24T13:55:20,980 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ee00117f0849,38621,1732456517828 WAL splitting is done? wals=0, meta=true 2024-11-24T13:55:20,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T13:55:20,983 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:20,984 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T13:55:21,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:21,060 INFO [RS:0;ee00117f0849:38621 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:21,060 INFO [RS:0;ee00117f0849:38621 {}] regionserver.HRegionServer(1031): Exiting; stopping=ee00117f0849,38621,1732456517828; zookeeper connection closed. 2024-11-24T13:55:21,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38621-0x1016e6a22f10001, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:21,061 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@228979f1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@228979f1 2024-11-24T13:55:21,134 DEBUG [ee00117f0849:33583 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-24T13:55:21,135 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,136 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,41583,1732456517869, state=OPENING 2024-11-24T13:55:21,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:21,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:21,161 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T13:55:21,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:21,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:21,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,41583,1732456517869}] 2024-11-24T13:55:21,211 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ee00117f0849:0 server-side Connection retries=45 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T13:55:21,211 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T13:55:21,212 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40345 2024-11-24T13:55:21,213 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40345 connecting to ZooKeeper ensemble=127.0.0.1:50150 2024-11-24T13:55:21,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:21,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:21,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403450x0, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T13:55:21,244 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40345-0x1016e6a22f10003 connected 2024-11-24T13:55:21,244 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-24T13:55:21,244 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-24T13:55:21,246 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T13:55:21,246 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache 2024-11-24T13:55:21,247 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-24T13:55:21,247 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-24T13:55:21,247 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-24T13:55:21,247 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-24T13:55:21,247 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-24T13:55:21,248 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T13:55:21,249 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-24T13:55:21,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T13:55:21,250 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-24T13:55:21,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T13:55:21,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T13:55:21,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40345 2024-11-24T13:55:21,254 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40345 2024-11-24T13:55:21,254 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-24T13:55:21,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T13:55:21,255 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40345 2024-11-24T13:55:21,255 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-24T13:55:21,255 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-24T13:55:21,255 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence is complete. 2024-11-24T13:55:21,256 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(746): ClusterId : 1ff47e21-5dfc-4dd2-bf4d-075d60034024 2024-11-24T13:55:21,256 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T13:55:21,261 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T13:55:21,261 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T13:55:21,269 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T13:55:21,270 DEBUG [RS:2;ee00117f0849:40345 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529a8647, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ee00117f0849/172.17.0.2:0 2024-11-24T13:55:21,285 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ee00117f0849:40345 2024-11-24T13:55:21,285 INFO [RS:2;ee00117f0849:40345 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T13:55:21,285 INFO [RS:2;ee00117f0849:40345 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T13:55:21,285 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T13:55:21,286 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(2659): reportForDuty to master=ee00117f0849,33583,1732456517700 with port=40345, startcode=1732456521210 2024-11-24T13:55:21,286 DEBUG [RS:2;ee00117f0849:40345 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T13:55:21,288 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41955, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T13:55:21,288 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,288 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] master.ServerManager(517): Registering regionserver=ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,290 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be 2024-11-24T13:55:21,290 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-24T13:55:21,290 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T13:55:21,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:21,302 DEBUG [RS:2;ee00117f0849:40345 {}] zookeeper.ZKUtil(111): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,302 WARN [RS:2;ee00117f0849:40345 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T13:55:21,302 INFO [RS:2;ee00117f0849:40345 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:21,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ee00117f0849,40345,1732456521210] 2024-11-24T13:55:21,303 DEBUG [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,309 INFO [RS:2;ee00117f0849:40345 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T13:55:21,313 INFO [RS:2;ee00117f0849:40345 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T13:55:21,313 INFO [RS:2;ee00117f0849:40345 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T13:55:21,313 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,314 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T13:55:21,314 INFO [RS:2;ee00117f0849:40345 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T13:55:21,314 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ee00117f0849:0, corePoolSize=2, maxPoolSize=2 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,315 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,316 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,316 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ee00117f0849:0, corePoolSize=1, maxPoolSize=1 2024-11-24T13:55:21,316 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:21,316 DEBUG [RS:2;ee00117f0849:40345 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ee00117f0849:0, corePoolSize=3, maxPoolSize=3 2024-11-24T13:55:21,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,317 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,40345,1732456521210-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T13:55:21,318 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T13:55:21,318 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T13:55:21,319 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T13:55:21,319 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C41583%2C1732456517869.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,41583,1732456517869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:21,333 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T13:55:21,333 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ee00117f0849,40345,1732456521210-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,333 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,333 INFO [RS:2;ee00117f0849:40345 {}] regionserver.Replication(171): ee00117f0849,40345,1732456521210 started 2024-11-24T13:55:21,336 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,41583,1732456517869/ee00117f0849%2C41583%2C1732456517869.meta.1732456521320.meta 2024-11-24T13:55:21,336 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:21,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:21,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T13:55:21,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T13:55:21,337 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T13:55:21,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T13:55:21,337 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:21,338 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T13:55:21,338 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T13:55:21,338 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-24T13:55:21,339 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T13:55:21,339 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:21,341 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975, delay=902 ms 2024-11-24T13:55:21,342 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:21,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:21,342 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-24T13:55:21,343 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T13:55:21,343 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:21,344 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e, delay=924 ms 2024-11-24T13:55:21,345 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:21,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:21,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-24T13:55:21,346 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T13:55:21,346 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:21,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:21,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-24T13:55:21,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T13:55:21,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:21,348 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4, delay=930 ms 2024-11-24T13:55:21,349 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:21,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T13:55:21,350 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T13:55:21,350 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T13:55:21,350 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1482): Serving as ee00117f0849,40345,1732456521210, RpcServer on ee00117f0849/172.17.0.2:40345, sessionid=0x1016e6a22f10003 2024-11-24T13:55:21,350 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T13:55:21,350 DEBUG [RS:2;ee00117f0849:40345 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,350 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,40345,1732456521210' 2024-11-24T13:55:21,350 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:21,350 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T13:55:21,351 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740 2024-11-24T13:55:21,351 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T13:55:21,351 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T13:55:21,351 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T13:55:21,352 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ee00117f0849,40345,1732456521210' 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T13:55:21,352 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T13:55:21,353 DEBUG [RS:2;ee00117f0849:40345 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T13:55:21,353 INFO [RS:2;ee00117f0849:40345 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T13:55:21,353 INFO [RS:2;ee00117f0849:40345 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T13:55:21,353 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T13:55:21,353 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69183276, jitterRate=0.030911147594451904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T13:55:21,353 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T13:55:21,354 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732456521338Writing region info on filesystem at 1732456521338Initializing all the Stores at 1732456521338Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456521338Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456521338Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456521338Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732456521338Cleaning up temporary data from old regions at 1732456521351 (+13 ms)Running coprocessor post-open hooks at 1732456521354 (+3 ms)Region opened successfully at 1732456521354 2024-11-24T13:55:21,355 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732456521314 2024-11-24T13:55:21,358 DEBUG [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T13:55:21,358 INFO [RS_OPEN_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T13:55:21,358 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,359 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ee00117f0849,41583,1732456517869, state=OPEN 2024-11-24T13:55:21,364 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33583 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ee00117f0849,41583,1732456517869, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-24T13:55:21,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:21,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:21,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T13:55:21,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:21,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:21,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T13:55:21,377 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-24T13:55:21,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ee00117f0849,41583,1732456517869 in 216 msec 2024-11-24T13:55:21,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T13:55:21,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 397 msec 2024-11-24T13:55:21,381 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): ee00117f0849,38621,1732456517828 had 2 regions 2024-11-24T13:55:21,382 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ee00117f0849,38621,1732456517828, splitWal=true, meta=true, isMeta: false 2024-11-24T13:55:21,382 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828-splitting dir is empty, no logs to split. 2024-11-24T13:55:21,383 INFO [PEWorker-3 {}] master.SplitWALManager(105): ee00117f0849,38621,1732456517828 WAL count=0, meta=false 2024-11-24T13:55:21,383 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,38621,1732456517828-splitting dir is empty, no logs to split. 2024-11-24T13:55:21,383 INFO [PEWorker-3 {}] master.SplitWALManager(105): ee00117f0849,38621,1732456517828 WAL count=0, meta=false 2024-11-24T13:55:21,384 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if ee00117f0849,38621,1732456517828 WAL splitting is done? wals=0, meta=false 2024-11-24T13:55:21,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN}] 2024-11-24T13:55:21,386 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN 2024-11-24T13:55:21,386 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T13:55:21,454 INFO [RS:2;ee00117f0849:40345 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T13:55:21,455 INFO [RS:2;ee00117f0849:40345 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ee00117f0849%2C40345%2C1732456521210, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,40345,1732456521210, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs, maxLogs=55 2024-11-24T13:55:21,476 INFO [RS:2;ee00117f0849:40345 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/WALs/ee00117f0849,40345,1732456521210/ee00117f0849%2C40345%2C1732456521210.1732456521456 2024-11-24T13:55:21,476 DEBUG [RS:2;ee00117f0849:40345 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(204): Hosts are {ee00117f0849=0} racks are {/default-rack=0} 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T13:55:21,537 INFO [ee00117f0849:33583 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T13:55:21,537 INFO [ee00117f0849:33583 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T13:55:21,537 DEBUG [ee00117f0849:33583 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T13:55:21,538 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7c8d18c3271e4f8f2397747fde461fca, regionState=OPENING, regionLocation=ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,539 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ee00117f0849:38621 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ee00117f0849/172.17.0.2:38621 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T13:55:21,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1, error=java.net.ConnectException: Call to address=ee00117f0849:38621 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ee00117f0849/172.17.0.2:38621 2024-11-24T13:55:21,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-24T13:55:21,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ee00117f0849,38621,1732456517828, seqNum=-1 from cache 2024-11-24T13:55:21,544 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address ee00117f0849:38621 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ee00117f0849/172.17.0.2:38621 2024-11-24T13:55:21,650 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T13:55:21,650 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ee00117f0849,41583,1732456517869, seqNum=-1] 2024-11-24T13:55:21,651 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T13:55:21,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33037, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T13:55:21,658 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN because future has completed 2024-11-24T13:55:21,660 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,41583,1732456517869}] 2024-11-24T13:55:21,816 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,817 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 7c8d18c3271e4f8f2397747fde461fca, NAME => 'testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.', STARTKEY => '', ENDKEY => ''} 2024-11-24T13:55:21,817 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732456519003 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,817 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T13:55:21,817 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,817 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,818 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,819 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7c8d18c3271e4f8f2397747fde461fca columnFamilyName family 2024-11-24T13:55:21,819 DEBUG [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T13:55:21,820 DEBUG [StoreFileOpener-7c8d18c3271e4f8f2397747fde461fca-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d, delay=979 ms 2024-11-24T13:55:21,821 DEBUG [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:21,821 INFO [StoreOpener-7c8d18c3271e4f8f2397747fde461fca-1 {}] regionserver.HStore(327): Store=7c8d18c3271e4f8f2397747fde461fca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T13:55:21,821 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,822 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,822 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,822 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,822 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,824 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,824 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 7c8d18c3271e4f8f2397747fde461fca; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73234924, jitterRate=0.09128540754318237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T13:55:21,824 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,825 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 7c8d18c3271e4f8f2397747fde461fca: Running coprocessor pre-open hook at 1732456521818Writing region info on filesystem at 1732456521818Initializing all the Stores at 1732456521818Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732456521818Cleaning up temporary data from old regions at 1732456521822 (+4 ms)Running coprocessor post-open hooks at 1732456521824 (+2 ms)Region opened successfully at 1732456521825 (+1 ms) 2024-11-24T13:55:21,826 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca., pid=11, masterSystemTime=1732456521814 2024-11-24T13:55:21,829 DEBUG [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,829 INFO [RS_OPEN_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,829 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=7c8d18c3271e4f8f2397747fde461fca, regionState=OPEN, openSeqNum=10, regionLocation=ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,41583,1732456517869 because future has completed 2024-11-24T13:55:21,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T13:55:21,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 7c8d18c3271e4f8f2397747fde461fca, server=ee00117f0849,41583,1732456517869 in 172 msec 2024-11-24T13:55:21,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-24T13:55:21,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732456519003, region=7c8d18c3271e4f8f2397747fde461fca, ASSIGN in 449 msec 2024-11-24T13:55:21,836 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server ee00117f0849,38621,1732456517828 after splitting done 2024-11-24T13:55:21,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure ee00117f0849,38621,1732456517828, splitWal=true, meta=true in 867 msec 2024-11-24T13:55:21,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T13:55:21,850 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:55:21,850 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:21,850 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,851 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,851 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T13:55:21,851 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T13:55:21,851 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1809726366, stopped=false 2024-11-24T13:55:21,851 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ee00117f0849,33583,1732456517700 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:21,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:21,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:55:21,894 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T13:55:21,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:21,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,895 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ee00117f0849,41583,1732456517869' ***** 2024-11-24T13:55:21,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:21,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:21,895 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:55:21,895 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ee00117f0849,40345,1732456521210' ***** 2024-11-24T13:55:21,895 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T13:55:21,895 INFO [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T13:55:21,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T13:55:21,896 INFO [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence 2024-11-24T13:55:21,896 INFO [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1738): Disabling cache 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 INFO [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/605b1a20-04b7-5a9f-b6b0-67ba083c2426/bucket.persistence 2024-11-24T13:55:21,896 INFO [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-24T13:55:21,896 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T13:55:21,897 INFO [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-24T13:55:21,897 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-24T13:55:21,906 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1487): Thread RS:1;ee00117f0849:41583 started persisting bucket cache to file 2024-11-24T13:55:21,906 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:21,906 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:21,907 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1487): Thread RS:2;ee00117f0849:40345 started persisting bucket cache to file 2024-11-24T13:55:21,907 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-24T13:55:21,907 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-24T13:55:21,909 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:21,909 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-24T13:55:21,909 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456521906 2024-11-24T13:55:21,909 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732456521907 2024-11-24T13:55:21,909 DEBUG [RS:1;ee00117f0849:41583 {}] bucket.BucketCache(1509): Thread RS:1;ee00117f0849:41583 finished persisting bucket cache to file, renaming 2024-11-24T13:55:21,909 DEBUG [RS:2;ee00117f0849:40345 {}] bucket.BucketCache(1509): Thread RS:2;ee00117f0849:40345 finished persisting bucket cache to file, renaming 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:55:21,910 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(3091): Received CLOSE for 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,910 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(959): stopping server ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(959): stopping server ee00117f0849,41583,1732456517869 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:21,910 INFO [RS:2;ee00117f0849:40345 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ee00117f0849:40345. 2024-11-24T13:55:21,910 INFO [RS:1;ee00117f0849:41583 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ee00117f0849:41583. 2024-11-24T13:55:21,910 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:21,910 DEBUG [RS:2;ee00117f0849:40345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:21,911 DEBUG [RS:2;ee00117f0849:40345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,911 DEBUG [RS:1;ee00117f0849:41583 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T13:55:21,911 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7c8d18c3271e4f8f2397747fde461fca, disabling compactions & flushes 2024-11-24T13:55:21,911 DEBUG [RS:1;ee00117f0849:41583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,911 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,911 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(976): stopping server ee00117f0849,40345,1732456521210; all regions closed. 2024-11-24T13:55:21,911 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,911 INFO [RS:1;ee00117f0849:41583 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:55:21,911 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. after waiting 0 ms 2024-11-24T13:55:21,911 INFO [RS:1;ee00117f0849:41583 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:55:21,911 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,911 INFO [RS:1;ee00117f0849:41583 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:55:21,911 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T13:55:21,911 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-24T13:55:21,911 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 7c8d18c3271e4f8f2397747fde461fca=testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca.} 2024-11-24T13:55:21,911 DEBUG [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 7c8d18c3271e4f8f2397747fde461fca 2024-11-24T13:55:21,911 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-24T13:55:21,911 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T13:55:21,911 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T13:55:21,912 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T13:55:21,912 DEBUG [StoreFileCloser-7c8d18c3271e4f8f2397747fde461fca-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/family/2c2b266a83fb4c2ebf3a6dfcc2db960d 2024-11-24T13:55:21,912 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T13:55:21,912 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T13:55:21,912 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-24T13:55:21,912 DEBUG [RS:2;ee00117f0849:40345 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C40345%2C1732456521210:(num 1732456521456) 2024-11-24T13:55:21,913 DEBUG [RS:2;ee00117f0849:40345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] hbase.ChoreService(370): Chore service for: regionserver/ee00117f0849:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T13:55:21,913 INFO [regionserver/ee00117f0849:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:21,913 INFO [RS:2;ee00117f0849:40345 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40345 2024-11-24T13:55:21,915 INFO [regionserver/ee00117f0849:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:21,916 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/default/testBlockEvictionOnGracefulStop1732456519003/7c8d18c3271e4f8f2397747fde461fca/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-24T13:55:21,916 INFO [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,916 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7c8d18c3271e4f8f2397747fde461fca: Waiting for close lock at 1732456521910Running coprocessor pre-close hooks at 1732456521911 (+1 ms)Disabling compacts and flushes for region at 1732456521911Disabling writes for close at 1732456521911Writing region close event to WAL at 1732456521912 (+1 ms)Running coprocessor post-close hooks at 1732456521916 (+4 ms)Closed at 1732456521916 2024-11-24T13:55:21,916 DEBUG [RS_CLOSE_REGION-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca. 2024-11-24T13:55:21,918 INFO [regionserver/ee00117f0849:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:21,920 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ddf54a2d9bc844bba3f8ee0d67e0729c 2024-11-24T13:55:21,922 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T13:55:21,922 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T13:55:21,924 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:55:21,924 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T13:55:21,924 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T13:55:21,924 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T13:55:21,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:21,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ee00117f0849,40345,1732456521210 2024-11-24T13:55:21,926 INFO [RS:2;ee00117f0849:40345 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:21,926 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ee00117f0849,40345,1732456521210] 2024-11-24T13:55:21,934 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ddf54a2d9bc844bba3f8ee0d67e0729c is 209, key is testBlockEvictionOnGracefulStop1732456519003,,1732456519007.7c8d18c3271e4f8f2397747fde461fca./info:regioninfo/1732456521829/Put/seqid=0 2024-11-24T13:55:21,935 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ddf54a2d9bc844bba3f8ee0d67e0729c 2024-11-24T13:55:21,937 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/.tmp/info/ddf54a2d9bc844bba3f8ee0d67e0729c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ddf54a2d9bc844bba3f8ee0d67e0729c 2024-11-24T13:55:21,938 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ddf54a2d9bc844bba3f8ee0d67e0729c, delay=930 ms 2024-11-24T13:55:21,939 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ddf54a2d9bc844bba3f8ee0d67e0729c, entries=8, sequenceid=19, filesize=6.6 K 2024-11-24T13:55:21,940 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false 2024-11-24T13:55:21,940 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ad5a15bc267543759a146c3ec8b4a975 2024-11-24T13:55:21,941 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/info/ddf54a2d9bc844bba3f8ee0d67e0729c 2024-11-24T13:55:21,941 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/ns/c7cac323b3f44120a34e7675b5df570e 2024-11-24T13:55:21,942 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/table/a9af9e8ef7a244edb4b6e1052c0fd3c4 2024-11-24T13:55:21,943 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ee00117f0849,40345,1732456521210 already deleted, retry=false 2024-11-24T13:55:21,943 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ee00117f0849,40345,1732456521210 expired; onlineServers=1 2024-11-24T13:55:21,945 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-24T13:55:21,946 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T13:55:21,946 INFO [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:21,946 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732456521911Running coprocessor pre-close hooks at 1732456521911Disabling compacts and flushes for region at 1732456521911Disabling writes for close at 1732456521912 (+1 ms)Obtaining lock to block concurrent updates at 1732456521912Preparing flush snapshotting stores in 1588230740 at 1732456521912Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732456521913 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732456521913Flushing 1588230740/info: creating writer at 1732456521913Flushing 1588230740/info: appending metadata at 1732456521934 (+21 ms)Flushing 1588230740/info: closing flushed file at 1732456521934Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74d798f4: reopening flushed file at 1732456521937 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 28ms, sequenceid=19, compaction requested=false at 1732456521940 (+3 ms)Writing region close event to WAL at 1732456521942 (+2 ms)Running coprocessor post-close hooks at 1732456521946 (+4 ms)Closed at 1732456521946 2024-11-24T13:55:21,946 DEBUG [RS_CLOSE_META-regionserver/ee00117f0849:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T13:55:22,035 INFO [RS:2;ee00117f0849:40345 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:22,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,035 INFO [RS:2;ee00117f0849:40345 {}] regionserver.HRegionServer(1031): Exiting; stopping=ee00117f0849,40345,1732456521210; zookeeper connection closed. 2024-11-24T13:55:22,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40345-0x1016e6a22f10003, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,036 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57f9387f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57f9387f 2024-11-24T13:55:22,112 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(976): stopping server ee00117f0849,41583,1732456517869; all regions closed. 2024-11-24T13:55:22,117 DEBUG [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:22,117 INFO [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C41583%2C1732456517869.meta:.meta(num 1732456521320) 2024-11-24T13:55:22,120 DEBUG [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/oldWALs 2024-11-24T13:55:22,120 INFO [RS:1;ee00117f0849:41583 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ee00117f0849%2C41583%2C1732456517869:(num 1732456518552) 2024-11-24T13:55:22,120 DEBUG [RS:1;ee00117f0849:41583 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T13:55:22,120 INFO [RS:1;ee00117f0849:41583 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T13:55:22,121 INFO [RS:1;ee00117f0849:41583 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:22,121 INFO [RS:1;ee00117f0849:41583 {}] hbase.ChoreService(370): Chore service for: regionserver/ee00117f0849:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:22,121 INFO [RS:1;ee00117f0849:41583 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:22,121 INFO [regionserver/ee00117f0849:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:22,122 INFO [RS:1;ee00117f0849:41583 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41583 2024-11-24T13:55:22,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T13:55:22,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ee00117f0849,41583,1732456517869 2024-11-24T13:55:22,143 INFO [RS:1;ee00117f0849:41583 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:22,144 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ee00117f0849,41583,1732456517869] 2024-11-24T13:55:22,160 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ee00117f0849,41583,1732456517869 already deleted, retry=false 2024-11-24T13:55:22,160 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ee00117f0849,41583,1732456517869 expired; onlineServers=0 2024-11-24T13:55:22,160 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ee00117f0849,33583,1732456517700' ***** 2024-11-24T13:55:22,160 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T13:55:22,160 INFO [M:0;ee00117f0849:33583 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T13:55:22,160 INFO [M:0;ee00117f0849:33583 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T13:55:22,160 DEBUG [M:0;ee00117f0849:33583 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T13:55:22,161 DEBUG [M:0;ee00117f0849:33583 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T13:55:22,161 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T13:55:22,161 DEBUG [master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456518239 {}] cleaner.HFileCleaner(306): Exit Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.large.0-1732456518239,5,FailOnTimeoutGroup] 2024-11-24T13:55:22,161 DEBUG [master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456518239 {}] cleaner.HFileCleaner(306): Exit Thread[master/ee00117f0849:0:becomeActiveMaster-HFileCleaner.small.0-1732456518239,5,FailOnTimeoutGroup] 2024-11-24T13:55:22,161 INFO [M:0;ee00117f0849:33583 {}] hbase.ChoreService(370): Chore service for: master/ee00117f0849:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T13:55:22,161 INFO [M:0;ee00117f0849:33583 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T13:55:22,161 DEBUG [M:0;ee00117f0849:33583 {}] master.HMaster(1795): Stopping service threads 2024-11-24T13:55:22,162 INFO [M:0;ee00117f0849:33583 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T13:55:22,162 INFO [M:0;ee00117f0849:33583 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T13:55:22,162 INFO [M:0;ee00117f0849:33583 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T13:55:22,162 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T13:55:22,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T13:55:22,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T13:55:22,168 DEBUG [M:0;ee00117f0849:33583 {}] zookeeper.ZKUtil(347): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T13:55:22,169 WARN [M:0;ee00117f0849:33583 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T13:55:22,169 INFO [M:0;ee00117f0849:33583 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/.lastflushedseqids 2024-11-24T13:55:22,177 INFO [M:0;ee00117f0849:33583 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T13:55:22,177 INFO [M:0;ee00117f0849:33583 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T13:55:22,177 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T13:55:22,177 INFO [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:22,178 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:22,178 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T13:55:22,178 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:22,178 INFO [M:0;ee00117f0849:33583 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-24T13:55:22,188 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7642696e982e4d23afba1c820e6a8f6d 2024-11-24T13:55:22,204 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7642696e982e4d23afba1c820e6a8f6d is 82, key is hbase:meta,,1/info:regioninfo/1732456521358/Put/seqid=0 2024-11-24T13:55:22,205 INFO [M:0;ee00117f0849:33583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7642696e982e4d23afba1c820e6a8f6d 2024-11-24T13:55:22,212 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ead43a5f9797489296db8c118301399a 2024-11-24T13:55:22,224 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ead43a5f9797489296db8c118301399a is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732456519392/Put/seqid=0 2024-11-24T13:55:22,225 INFO [M:0;ee00117f0849:33583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ead43a5f9797489296db8c118301399a 2024-11-24T13:55:22,231 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d664625c9b01431c97a3b5c9526741d1 2024-11-24T13:55:22,244 DEBUG [M:0;ee00117f0849:33583 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d664625c9b01431c97a3b5c9526741d1 is 69, key is ee00117f0849,40345,1732456521210/rs:state/1732456521289/Put/seqid=0 2024-11-24T13:55:22,244 INFO [M:0;ee00117f0849:33583 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d664625c9b01431c97a3b5c9526741d1 2024-11-24T13:55:22,246 INFO [M:0;ee00117f0849:33583 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d664625c9b01431c97a3b5c9526741d1 2024-11-24T13:55:22,246 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7642696e982e4d23afba1c820e6a8f6d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7642696e982e4d23afba1c820e6a8f6d 2024-11-24T13:55:22,248 INFO [M:0;ee00117f0849:33583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7642696e982e4d23afba1c820e6a8f6d, entries=14, sequenceid=102, filesize=5.9 K 2024-11-24T13:55:22,248 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ead43a5f9797489296db8c118301399a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ead43a5f9797489296db8c118301399a 2024-11-24T13:55:22,249 INFO [M:0;ee00117f0849:33583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ead43a5f9797489296db8c118301399a, entries=11, sequenceid=102, filesize=6.5 K 2024-11-24T13:55:22,250 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d664625c9b01431c97a3b5c9526741d1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d664625c9b01431c97a3b5c9526741d1 2024-11-24T13:55:22,251 INFO [M:0;ee00117f0849:33583 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d664625c9b01431c97a3b5c9526741d1 2024-11-24T13:55:22,251 INFO [M:0;ee00117f0849:33583 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1bc55f98-3e3e-8732-4e95-fa8efecea737/5178e1ec-3006-5345-e3e6-6d24a33f66be/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d664625c9b01431c97a3b5c9526741d1, entries=3, sequenceid=102, filesize=5.3 K 2024-11-24T13:55:22,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,252 INFO [M:0;ee00117f0849:33583 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=102, compaction requested=false 2024-11-24T13:55:22,252 INFO [RS:1;ee00117f0849:41583 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:22,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41583-0x1016e6a22f10002, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,252 INFO [RS:1;ee00117f0849:41583 {}] regionserver.HRegionServer(1031): Exiting; stopping=ee00117f0849,41583,1732456517869; zookeeper connection closed. 2024-11-24T13:55:22,252 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@24745f54 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@24745f54 2024-11-24T13:55:22,252 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T13:55:22,253 INFO [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T13:55:22,254 DEBUG [M:0;ee00117f0849:33583 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732456522177Disabling compacts and flushes for region at 1732456522177Disabling writes for close at 1732456522178 (+1 ms)Obtaining lock to block concurrent updates at 1732456522178Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732456522178Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1732456522178Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732456522179 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732456522179Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732456522204 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732456522204Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732456522207 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732456522224 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732456522224Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732456522226 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732456522243 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732456522243Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2730c9c2: reopening flushed file at 1732456522246 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@159b3693: reopening flushed file at 1732456522248 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20f8f75b: reopening flushed file at 1732456522249 (+1 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=102, compaction requested=false at 1732456522252 (+3 ms)Writing region close event to WAL at 1732456522253 (+1 ms)Closed at 1732456522253 2024-11-24T13:55:22,255 INFO [M:0;ee00117f0849:33583 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T13:55:22,255 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T13:55:22,255 INFO [M:0;ee00117f0849:33583 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33583 2024-11-24T13:55:22,255 INFO [M:0;ee00117f0849:33583 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T13:55:22,368 INFO [M:0;ee00117f0849:33583 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T13:55:22,368 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33583-0x1016e6a22f10000, quorum=127.0.0.1:50150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T13:55:22,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T13:55:22,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-24T13:55:22,376 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T13:55:22,383 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=200 (was 217), ProcessCount=11 (was 11), AvailableMemoryMB=8326 (was 8402)