2024-11-20 02:54:56,574 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 02:54:56,595 main DEBUG Took 0.013348 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 02:54:56,595 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 02:54:56,595 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 02:54:56,597 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 02:54:56,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,607 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 02:54:56,621 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,623 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,624 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,624 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,625 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,626 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,627 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,628 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,628 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,629 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,630 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,631 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,632 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,632 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,633 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,633 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,634 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,634 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,635 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,636 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,636 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,637 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,638 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,638 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 02:54:56,639 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,639 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 02:54:56,642 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 02:54:56,643 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 02:54:56,646 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 02:54:56,647 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 02:54:56,649 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 02:54:56,649 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 02:54:56,659 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 02:54:56,663 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 02:54:56,665 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 02:54:56,666 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 02:54:56,667 main DEBUG createAppenders(={Console}) 2024-11-20 02:54:56,668 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 02:54:56,668 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 02:54:56,669 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 02:54:56,669 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 02:54:56,670 main DEBUG OutputStream closed 2024-11-20 02:54:56,670 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 02:54:56,671 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 02:54:56,671 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 02:54:56,769 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 02:54:56,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 02:54:56,773 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 02:54:56,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 02:54:56,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 02:54:56,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 02:54:56,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 02:54:56,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 02:54:56,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 02:54:56,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 02:54:56,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 02:54:56,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 02:54:56,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 02:54:56,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 02:54:56,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 02:54:56,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 02:54:56,780 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 02:54:56,781 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 02:54:56,784 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 02:54:56,784 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 02:54:56,785 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 02:54:56,786 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T02:54:56,802 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 02:54:56,806 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 02:54:56,806 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T02:54:57,110 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e 2024-11-20T02:54:57,135 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=356, ProcessCount=11, AvailableMemoryMB=7559 2024-11-20T02:54:57,218 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T02:54:57,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:57,321 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/cluster_6f105dc3-c77d-8a04-f6e6-b3d8656326df, deleteOnExit=true 2024-11-20T02:54:57,414 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/cluster_6f105dc3-c77d-8a04-f6e6-b3d8656326df/zookeeper_0, clientPort=54683, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/cluster_6f105dc3-c77d-8a04-f6e6-b3d8656326df/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/cluster_6f105dc3-c77d-8a04-f6e6-b3d8656326df/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T02:54:57,427 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54683 2024-11-20T02:54:57,428 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:57,514 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 with version=8 2024-11-20T02:54:57,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase-staging 2024-11-20T02:54:57,610 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T02:54:57,923 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:54:57,932 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:57,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:57,946 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:54:57,946 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:57,947 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:54:58,136 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T02:54:58,219 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:54:58,270 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T02:54:58,286 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T02:54:58,292 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:54:58,346 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 95788 (auto-detected) 2024-11-20T02:54:58,347 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T02:54:58,371 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40025 2024-11-20T02:54:58,401 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40025 connecting to ZooKeeper ensemble=127.0.0.1:54683 2024-11-20T02:54:58,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:400250x0, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:54:58,500 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40025-0x101577424340000 connected 2024-11-20T02:54:58,731 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:58,732 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:58,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:54:58,745 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98, hbase.cluster.distributed=false 2024-11-20T02:54:58,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:54:58,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-20T02:54:58,779 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40025 2024-11-20T02:54:58,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40025 2024-11-20T02:54:58,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-20T02:54:58,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40025 2024-11-20T02:54:58,865 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:54:58,866 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:58,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:58,867 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:54:58,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:58,867 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:54:58,870 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:54:58,874 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:54:58,876 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42601 2024-11-20T02:54:58,878 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42601 connecting to ZooKeeper ensemble=127.0.0.1:54683 2024-11-20T02:54:58,878 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:58,879 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:58,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426010x0, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:54:58,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42601-0x101577424340001 connected 2024-11-20T02:54:58,903 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:54:58,912 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:54:58,932 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.cache 2024-11-20T02:54:58,940 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T02:54:58,942 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T02:54:58,944 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T02:54:58,948 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T02:54:58,948 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T02:54:58,956 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T02:54:58,960 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T02:54:58,960 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence is complete. 2024-11-20T02:54:58,962 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:54:58,965 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T02:54:58,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:54:58,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42601 2024-11-20T02:54:58,975 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42601 2024-11-20T02:54:58,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42601 2024-11-20T02:54:58,981 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42601 2024-11-20T02:54:58,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42601 2024-11-20T02:54:59,008 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:54:59,008 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:59,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:59,009 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:54:59,009 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:54:59,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:54:59,010 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:54:59,011 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:54:59,014 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44699 2024-11-20T02:54:59,017 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44699 connecting to ZooKeeper ensemble=127.0.0.1:54683 2024-11-20T02:54:59,018 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:59,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:59,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446990x0, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:54:59,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:54:59,052 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:54:59,050 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44699-0x101577424340002 connected 2024-11-20T02:54:59,058 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.cache 2024-11-20T02:54:59,059 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T02:54:59,059 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T02:54:59,060 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T02:54:59,061 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T02:54:59,061 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T02:54:59,062 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T02:54:59,062 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence is complete. 2024-11-20T02:54:59,063 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T02:54:59,068 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:54:59,071 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T02:54:59,074 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:54:59,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44699 2024-11-20T02:54:59,081 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44699 2024-11-20T02:54:59,082 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44699 2024-11-20T02:54:59,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44699 2024-11-20T02:54:59,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44699 2024-11-20T02:54:59,114 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1c82b83631a2:40025 2024-11-20T02:54:59,117 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1c82b83631a2,40025,1732071297729 2024-11-20T02:54:59,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,151 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1c82b83631a2,40025,1732071297729 2024-11-20T02:54:59,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T02:54:59,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T02:54:59,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,230 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T02:54:59,232 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1c82b83631a2,40025,1732071297729 from backup master directory 2024-11-20T02:54:59,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1c82b83631a2,40025,1732071297729 2024-11-20T02:54:59,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,248 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:54:59,246 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:54:59,249 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1c82b83631a2,40025,1732071297729 2024-11-20T02:54:59,252 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T02:54:59,255 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T02:54:59,268 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase.id] with ID: 6d1223c9-532b-40da-9c21-0e43fd99de2d 2024-11-20T02:54:59,268 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.tmp/hbase.id 2024-11-20T02:54:59,275 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase.id] 2024-11-20T02:54:59,335 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:54:59,340 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T02:54:59,360 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-20T02:54:59,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:54:59,411 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:54:59,414 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T02:54:59,458 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T02:54:59,464 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:54:59,524 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store 2024-11-20T02:54:59,545 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T02:54:59,548 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:54:59,550 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:54:59,550 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:54:59,551 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:54:59,552 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:54:59,552 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:54:59,553 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:54:59,553 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071299550Disabling compacts and flushes for region at 1732071299550Disabling writes for close at 1732071299552 (+2 ms)Writing region close event to WAL at 1732071299552Closed at 1732071299552 2024-11-20T02:54:59,554 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/.initializing 2024-11-20T02:54:59,554 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,40025,1732071297729 2024-11-20T02:54:59,566 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:54:59,577 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C40025%2C1732071297729, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,40025,1732071297729, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/oldWALs, maxLogs=10 2024-11-20T02:54:59,611 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,40025,1732071297729/1c82b83631a2%2C40025%2C1732071297729.1732071299581 2024-11-20T02:54:59,612 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:54:59,613 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:54:59,613 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:54:59,617 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,618 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,685 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T02:54:59,691 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:54:59,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:54:59,693 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T02:54:59,698 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:54:59,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:54:59,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,704 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T02:54:59,704 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:54:59,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:54:59,705 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T02:54:59,709 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:54:59,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:54:59,710 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,713 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,714 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,718 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,718 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,722 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:54:59,724 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:54:59,739 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:54:59,741 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74860050, jitterRate=0.11550167202949524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:54:59,752 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732071299630Initializing all the Stores at 1732071299630Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071299631 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071299631Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071299632 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071299632Cleaning up temporary data from old regions at 1732071299719 (+87 ms)Region opened successfully at 1732071299751 (+32 ms) 2024-11-20T02:54:59,754 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T02:54:59,785 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66db390b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:54:59,816 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T02:54:59,828 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T02:54:59,828 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T02:54:59,832 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T02:54:59,833 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T02:54:59,838 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T02:54:59,839 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T02:54:59,864 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T02:54:59,873 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T02:54:59,924 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T02:54:59,926 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T02:54:59,928 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T02:54:59,940 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T02:54:59,942 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T02:54:59,946 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T02:54:59,957 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T02:54:59,958 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T02:54:59,969 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T02:54:59,985 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T02:54:59,994 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,019 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1c82b83631a2,40025,1732071297729, sessionid=0x101577424340000, setting cluster-up flag (Was=false) 2024-11-20T02:55:00,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,068 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T02:55:00,071 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,40025,1732071297729 2024-11-20T02:55:00,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,118 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T02:55:00,120 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,40025,1732071297729 2024-11-20T02:55:00,125 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T02:55:00,193 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(746): ClusterId : 6d1223c9-532b-40da-9c21-0e43fd99de2d 2024-11-20T02:55:00,193 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(746): ClusterId : 6d1223c9-532b-40da-9c21-0e43fd99de2d 2024-11-20T02:55:00,196 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:55:00,196 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:55:00,198 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:00,208 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T02:55:00,218 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T02:55:00,225 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:55:00,225 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:55:00,225 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:55:00,225 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:55:00,224 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1c82b83631a2,40025,1732071297729 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T02:55:00,237 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:55:00,237 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:55:00,238 DEBUG [RS:1;1c82b83631a2:44699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@759e41af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:00,238 DEBUG [RS:0;1c82b83631a2:42601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1d1691, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:00,238 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:00,239 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:00,239 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:00,239 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:00,239 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1c82b83631a2:0, corePoolSize=10, maxPoolSize=10 2024-11-20T02:55:00,240 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,240 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:00,240 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,246 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071330246 2024-11-20T02:55:00,247 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:00,248 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T02:55:00,249 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T02:55:00,250 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T02:55:00,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T02:55:00,254 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:00,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T02:55:00,255 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T02:55:00,255 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T02:55:00,254 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T02:55:00,256 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,260 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1c82b83631a2:44699 2024-11-20T02:55:00,263 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T02:55:00,264 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:55:00,264 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:55:00,264 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:55:00,264 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T02:55:00,265 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1c82b83631a2:42601 2024-11-20T02:55:00,265 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T02:55:00,265 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:55:00,265 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:55:00,265 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:55:00,267 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,40025,1732071297729 with port=44699, startcode=1732071299007 2024-11-20T02:55:00,267 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,40025,1732071297729 with port=42601, startcode=1732071298829 2024-11-20T02:55:00,268 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T02:55:00,269 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T02:55:00,275 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071300270,5,FailOnTimeoutGroup] 2024-11-20T02:55:00,276 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071300275,5,FailOnTimeoutGroup] 2024-11-20T02:55:00,277 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,277 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T02:55:00,278 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,278 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,282 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T02:55:00,283 DEBUG [RS:1;1c82b83631a2:44699 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:55:00,283 DEBUG [RS:0;1c82b83631a2:42601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:55:00,283 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:00,296 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:00,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T02:55:00,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:55:00,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:00,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:00,305 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T02:55:00,310 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:55:00,310 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:00,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:00,311 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T02:55:00,323 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:55:00,323 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:00,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:00,324 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T02:55:00,356 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:55:00,356 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:00,357 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:00,357 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:55:00,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:00,359 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:00,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:55:00,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:55:00,361 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:55:00,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:55:00,366 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:55:00,366 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40547, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:55:00,376 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40025 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,378 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:55:00,381 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75344185, jitterRate=0.12271584570407867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:55:00,381 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40025 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732071300296Initializing all the Stores at 1732071300297 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071300297Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071300298 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071300298Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071300298Cleaning up temporary data from old regions at 1732071300360 (+62 ms)Region opened successfully at 1732071300386 (+26 ms) 2024-11-20T02:55:00,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:55:00,387 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:55:00,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:55:00,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:55:00,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:55:00,389 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:00,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071300387Disabling compacts and flushes for region at 1732071300387Disabling writes for close at 1732071300388 (+1 ms)Writing region close event to WAL at 1732071300389 (+1 ms)Closed at 1732071300389 2024-11-20T02:55:00,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:00,393 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T02:55:00,397 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40025 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,397 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40025 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,402 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:00,402 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T02:55:00,402 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:55:00,406 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:00,406 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T02:55:00,407 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:55:00,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T02:55:00,416 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:55:00,418 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T02:55:00,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:00,483 DEBUG [RS:1;1c82b83631a2:44699 {}] zookeeper.ZKUtil(111): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,483 WARN [RS:1;1c82b83631a2:44699 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:00,483 INFO [RS:1;1c82b83631a2:44699 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:00,484 DEBUG [RS:0;1c82b83631a2:42601 {}] zookeeper.ZKUtil(111): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,484 WARN [RS:0;1c82b83631a2:42601 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:00,484 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,484 INFO [RS:0;1c82b83631a2:42601 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:00,484 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,44699,1732071299007] 2024-11-20T02:55:00,486 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,42601,1732071298829] 2024-11-20T02:55:00,511 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:55:00,514 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:55:00,531 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:55:00,531 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:55:00,537 INFO [RS:0;1c82b83631a2:42601 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:55:00,537 INFO [RS:1;1c82b83631a2:44699 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:55:00,538 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,538 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,539 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:55:00,539 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:55:00,546 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:55:00,546 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:55:00,549 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,550 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,550 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,550 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,550 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,551 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:00,552 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,552 DEBUG [RS:1;1c82b83631a2:44699 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:00,553 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:00,553 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:00,553 DEBUG [RS:0;1c82b83631a2:42601 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:00,565 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,566 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,566 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,566 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,567 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,42601,1732071298829-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:00,568 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,568 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,44699,1732071299007-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:00,570 WARN [1c82b83631a2:40025 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:55:00,593 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:55:00,593 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:55:00,595 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,44699,1732071299007-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,595 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,42601,1732071298829-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,595 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,595 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,596 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.Replication(171): 1c82b83631a2,44699,1732071299007 started 2024-11-20T02:55:00,596 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.Replication(171): 1c82b83631a2,42601,1732071298829 started 2024-11-20T02:55:00,622 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,622 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:00,622 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,44699,1732071299007, RpcServer on 1c82b83631a2/172.17.0.2:44699, sessionid=0x101577424340002 2024-11-20T02:55:00,622 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,42601,1732071298829, RpcServer on 1c82b83631a2/172.17.0.2:42601, sessionid=0x101577424340001 2024-11-20T02:55:00,623 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:55:00,623 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:55:00,624 DEBUG [RS:1;1c82b83631a2:44699 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,624 DEBUG [RS:0;1c82b83631a2:42601 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,624 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,42601,1732071298829' 2024-11-20T02:55:00,624 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,44699,1732071299007' 2024-11-20T02:55:00,624 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:55:00,624 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:55:00,625 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:55:00,625 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:55:00,626 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:55:00,626 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:55:00,627 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:55:00,627 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:55:00,627 DEBUG [RS:1;1c82b83631a2:44699 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,44699,1732071299007 2024-11-20T02:55:00,627 DEBUG [RS:0;1c82b83631a2:42601 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,627 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,44699,1732071299007' 2024-11-20T02:55:00,627 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,42601,1732071298829' 2024-11-20T02:55:00,627 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:55:00,627 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:55:00,628 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:55:00,628 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:55:00,629 DEBUG [RS:1;1c82b83631a2:44699 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:55:00,629 DEBUG [RS:0;1c82b83631a2:42601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:55:00,629 INFO [RS:1;1c82b83631a2:44699 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:55:00,629 INFO [RS:0;1c82b83631a2:42601 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:55:00,629 INFO [RS:1;1c82b83631a2:44699 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:55:00,629 INFO [RS:0;1c82b83631a2:42601 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:55:00,734 INFO [RS:0;1c82b83631a2:42601 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:00,734 INFO [RS:1;1c82b83631a2:44699 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:00,735 INFO [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C42601%2C1732071298829, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,42601,1732071298829, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:00,735 INFO [RS:1;1c82b83631a2:44699 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C44699%2C1732071299007, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,44699,1732071299007, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:00,756 INFO [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,42601,1732071298829/1c82b83631a2%2C42601%2C1732071298829.1732071300738 2024-11-20T02:55:00,758 DEBUG [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:00,763 INFO [RS:1;1c82b83631a2:44699 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,44699,1732071299007/1c82b83631a2%2C44699%2C1732071299007.1732071300738 2024-11-20T02:55:00,764 DEBUG [RS:1;1c82b83631a2:44699 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:00,822 DEBUG [1c82b83631a2:40025 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T02:55:00,837 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T02:55:00,841 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T02:55:00,841 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T02:55:00,841 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T02:55:00,841 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T02:55:00,841 INFO [1c82b83631a2:40025 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T02:55:00,841 INFO [1c82b83631a2:40025 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T02:55:00,841 DEBUG [1c82b83631a2:40025 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T02:55:00,848 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:00,854 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42601,1732071298829, state=OPENING 2024-11-20T02:55:00,868 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T02:55:00,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:00,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:00,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:00,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:00,880 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:55:00,882 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42601,1732071298829}] 2024-11-20T02:55:01,056 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T02:55:01,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T02:55:01,067 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T02:55:01,067 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:01,068 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T02:55:01,070 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C42601%2C1732071298829.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,42601,1732071298829, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:01,091 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,42601,1732071298829/1c82b83631a2%2C42601%2C1732071298829.meta.1732071301073.meta 2024-11-20T02:55:01,092 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:01,092 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:01,094 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T02:55:01,096 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T02:55:01,100 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T02:55:01,103 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T02:55:01,104 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:01,104 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T02:55:01,104 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T02:55:01,106 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T02:55:01,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:55:01,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:01,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T02:55:01,108 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:55:01,108 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:01,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T02:55:01,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:55:01,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:01,109 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T02:55:01,110 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:55:01,110 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,110 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:01,110 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:55:01,112 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:01,112 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:01,113 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:55:01,113 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:55:01,113 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:55:01,114 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:55:01,114 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60474624, jitterRate=-0.09885787963867188}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:55:01,115 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T02:55:01,116 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732071301105Writing region info on filesystem at 1732071301105Initializing all the Stores at 1732071301105Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071301105Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071301106 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071301106Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071301106Cleaning up temporary data from old regions at 1732071301113 (+7 ms)Running coprocessor post-open hooks at 1732071301115 (+2 ms)Region opened successfully at 1732071301116 (+1 ms) 2024-11-20T02:55:01,122 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732071301047 2024-11-20T02:55:01,130 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T02:55:01,131 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T02:55:01,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:01,133 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,42601,1732071298829, state=OPEN 2024-11-20T02:55:01,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:01,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:01,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:01,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:01,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:01,161 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:01,161 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:01,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T02:55:01,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,42601,1732071298829 in 279 msec 2024-11-20T02:55:01,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T02:55:01,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 761 msec 2024-11-20T02:55:01,173 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:01,173 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T02:55:01,192 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:55:01,194 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,42601,1732071298829, seqNum=-1] 2024-11-20T02:55:01,210 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:55:01,213 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59151, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:55:01,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0730 sec 2024-11-20T02:55:01,231 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732071301231, completionTime=-1 2024-11-20T02:55:01,233 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T02:55:01,233 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T02:55:01,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T02:55:01,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071361254 2024-11-20T02:55:01,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071421254 2024-11-20T02:55:01,254 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-11-20T02:55:01,257 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T02:55:01,264 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,265 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,265 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,266 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1c82b83631a2:40025, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,267 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,267 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,273 DEBUG [master/1c82b83631a2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T02:55:01,305 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.056sec 2024-11-20T02:55:01,307 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T02:55:01,308 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T02:55:01,309 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T02:55:01,309 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T02:55:01,310 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T02:55:01,310 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:01,311 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T02:55:01,312 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T02:55:01,327 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T02:55:01,328 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,40025,1732071297729-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:01,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f731eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:01,404 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T02:55:01,404 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T02:55:01,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1c82b83631a2,40025,-1 for getting cluster id 2024-11-20T02:55:01,411 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T02:55:01,419 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6d1223c9-532b-40da-9c21-0e43fd99de2d' 2024-11-20T02:55:01,421 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T02:55:01,421 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6d1223c9-532b-40da-9c21-0e43fd99de2d" 2024-11-20T02:55:01,422 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3819f4b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:01,422 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1c82b83631a2,40025,-1] 2024-11-20T02:55:01,424 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T02:55:01,426 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:01,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54310, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T02:55:01,430 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567d9853, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:01,431 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:55:01,438 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,42601,1732071298829, seqNum=-1] 2024-11-20T02:55:01,438 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:55:01,441 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:55:01,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1c82b83631a2,40025,1732071297729 2024-11-20T02:55:01,463 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T02:55:01,467 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1c82b83631a2,40025,1732071297729 2024-11-20T02:55:01,469 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fbe144e 2024-11-20T02:55:01,470 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T02:55:01,473 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T02:55:01,478 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732071301460', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:55:01,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 2024-11-20T02:55:01,489 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:55:01,491 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,491 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732071301460" procId is: 4 2024-11-20T02:55:01,493 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:55:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:01,504 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f04567280baab08e1176cbf031dac46b, NAME => 'testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732071301460', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:01,511 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:01,511 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1722): Closing f04567280baab08e1176cbf031dac46b, disabling compactions & flushes 2024-11-20T02:55:01,511 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,511 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,511 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. after waiting 0 ms 2024-11-20T02:55:01,511 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,512 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,512 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732071301460-pool-0 {}] regionserver.HRegion(1676): Region close journal for f04567280baab08e1176cbf031dac46b: Waiting for close lock at 1732071301511Disabling compacts and flushes for region at 1732071301511Disabling writes for close at 1732071301511Writing region close event to WAL at 1732071301511Closed at 1732071301511 2024-11-20T02:55:01,514 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:55:01,519 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732071301514"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071301514"}]},"ts":"1732071301514"} 2024-11-20T02:55:01,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:55:01,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:55:01,529 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732071301460","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071301526"}]},"ts":"1732071301526"} 2024-11-20T02:55:01,533 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732071301460, state=ENABLING in hbase:meta 2024-11-20T02:55:01,533 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T02:55:01,534 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T02:55:01,534 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T02:55:01,534 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T02:55:01,534 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T02:55:01,534 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T02:55:01,534 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T02:55:01,534 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T02:55:01,536 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, ASSIGN}] 2024-11-20T02:55:01,537 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, ASSIGN 2024-11-20T02:55:01,538 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, ASSIGN; state=OFFLINE, location=1c82b83631a2,42601,1732071298829; forceNewPlan=false, retain=false 2024-11-20T02:55:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:01,696 INFO [1c82b83631a2:40025 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T02:55:01,698 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=OPENING, regionLocation=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:01,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, ASSIGN because future has completed 2024-11-20T02:55:01,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829}] 2024-11-20T02:55:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:01,862 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,863 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f04567280baab08e1176cbf031dac46b, NAME => 'testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:01,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732071301460 f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:01,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,864 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,866 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,872 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f04567280baab08e1176cbf031dac46b columnFamilyName family 2024-11-20T02:55:01,872 DEBUG [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:01,873 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(327): Store=f04567280baab08e1176cbf031dac46b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:01,873 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,875 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,875 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,875 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,875 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,877 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,883 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:55:01,883 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f04567280baab08e1176cbf031dac46b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70307521, jitterRate=0.04766370356082916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:55:01,883 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:01,884 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f04567280baab08e1176cbf031dac46b: Running coprocessor pre-open hook at 1732071301865Writing region info on filesystem at 1732071301865Initializing all the Stores at 1732071301865Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071301866 (+1 ms)Cleaning up temporary data from old regions at 1732071301875 (+9 ms)Running coprocessor post-open hooks at 1732071301883 (+8 ms)Region opened successfully at 1732071301884 (+1 ms) 2024-11-20T02:55:01,885 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b., pid=6, masterSystemTime=1732071301856 2024-11-20T02:55:01,888 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,888 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:01,888 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:01,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829 because future has completed 2024-11-20T02:55:01,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T02:55:01,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829 in 190 msec 2024-11-20T02:55:01,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T02:55:01,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, ASSIGN in 361 msec 2024-11-20T02:55:01,900 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:55:01,900 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732071301460","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071301900"}]},"ts":"1732071301900"} 2024-11-20T02:55:01,903 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732071301460, state=ENABLED in hbase:meta 2024-11-20T02:55:01,904 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:55:01,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732071301460 in 422 msec 2024-11-20T02:55:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:02,126 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732071301460 get assigned. Timeout = 60000ms 2024-11-20T02:55:02,126 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732071301460 completed 2024-11-20T02:55:02,131 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:55:02,137 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732071301460 assigned to meta. Checking AM states. 2024-11-20T02:55:02,138 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:55:02,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732071301460 assigned. 2024-11-20T02:55:02,146 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732071301460', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b., hostname=1c82b83631a2,42601,1732071298829, seqNum=2] 2024-11-20T02:55:02,156 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f04567280baab08e1176cbf031dac46b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T02:55:02,179 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:02,213 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/.tmp/family/b70f76003ce646baaa79c386fda8bdfd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T02:55:02,218 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/.tmp/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:02,251 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/.tmp/family/b70f76003ce646baaa79c386fda8bdfd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:02,252 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, delay=976 ms 2024-11-20T02:55:02,254 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T02:55:02,259 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f04567280baab08e1176cbf031dac46b in 103ms, sequenceid=6, compaction requested=false 2024-11-20T02:55:02,260 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732071301460' 2024-11-20T02:55:02,262 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f04567280baab08e1176cbf031dac46b: 2024-11-20T02:55:03,230 DEBUG [hfile-prefetch-1732071302252 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b70f76003ce646baaa79c386fda8bdfd_0. 2024-11-20T02:55:03,233 DEBUG [hfile-prefetch-1732071302252 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b70f76003ce646baaa79c386fda8bdfd_117. 2024-11-20T02:55:03,234 DEBUG [hfile-prefetch-1732071302252 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, with total blocks 2, and data blocks 1 2024-11-20T02:55:03,234 DEBUG [hfile-prefetch-1732071302252 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T02:55:03,235 DEBUG [hfile-prefetch-1732071302252 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T02:55:03,235 DEBUG [hfile-prefetch-1732071302252 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd. Will sleep 100ms and try the verification again. 2024-11-20T02:55:03,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f04567280baab08e1176cbf031dac46b, source=1c82b83631a2,42601,1732071298829, destination=1c82b83631a2,44699,1732071299007, warming up region on 1c82b83631a2,44699,1732071299007 2024-11-20T02:55:03,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T02:55:03,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f04567280baab08e1176cbf031dac46b, source=1c82b83631a2,42601,1732071298829, destination=1c82b83631a2,44699,1732071299007, running balancer 2024-11-20T02:55:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE 2024-11-20T02:55:03,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35997, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T02:55:03,281 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE 2024-11-20T02:55:03,283 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=CLOSING, regionLocation=1c82b83631a2,42601,1732071298829 2024-11-20T02:55:03,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(7855): Warmup {ENCODED => f04567280baab08e1176cbf031dac46b, NAME => 'testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:03,284 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,285 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f04567280baab08e1176cbf031dac46b columnFamilyName family 2024-11-20T02:55:03,285 DEBUG [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:03,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE because future has completed 2024-11-20T02:55:03,286 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T02:55:03,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829}] 2024-11-20T02:55:03,289 DEBUG [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:03,289 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(327): Store=f04567280baab08e1176cbf031dac46b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1722): Closing f04567280baab08e1176cbf031dac46b, disabling compactions & flushes 2024-11-20T02:55:03,290 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. after waiting 0 ms 2024-11-20T02:55:03,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,291 DEBUG [StoreFileCloser-f04567280baab08e1176cbf031dac46b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:03,292 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44699 {}] regionserver.HRegion(1676): Region close journal for f04567280baab08e1176cbf031dac46b: Waiting for close lock at 1732071303290Disabling compacts and flushes for region at 1732071303290Disabling writes for close at 1732071303290Writing region close event to WAL at 1732071303292 (+2 ms)Closed at 1732071303292 2024-11-20T02:55:03,293 WARN [hfile-prefetch-1732071302252 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T02:55:03,294 DEBUG [hfile-prefetch-1732071302252 {}] hfile.PrefetchExecutor(121): Prefetch completed for b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:03,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40025 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T02:55:03,350 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T02:55:03,350 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T02:55:03,350 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,350 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,364 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T02:55:03,364 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T02:55:03,364 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,364 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,368 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:03,368 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071303364 2024-11-20T02:55:03,368 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,370 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T02:55:03,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732071303350 2024-11-20T02:55:03,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,446 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,446 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T02:55:03,447 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f04567280baab08e1176cbf031dac46b, disabling compactions & flushes 2024-11-20T02:55:03,447 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,448 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,448 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. after waiting 0 ms 2024-11-20T02:55:03,448 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,450 DEBUG [StoreFileCloser-f04567280baab08e1176cbf031dac46b-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b70f76003ce646baaa79c386fda8bdfd, starting offset: 0, end offset: 9223372036854775807 2024-11-20T02:55:03,452 DEBUG [StoreFileCloser-f04567280baab08e1176cbf031dac46b-family-1 {}] bucket.BucketCache(856): removed key b70f76003ce646baaa79c386fda8bdfd_0 from back map with offset lock 2094080 in the evict process 2024-11-20T02:55:03,459 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T02:55:03,461 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,461 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f04567280baab08e1176cbf031dac46b: Waiting for close lock at 1732071303447Running coprocessor pre-close hooks at 1732071303447Disabling compacts and flushes for region at 1732071303447Disabling writes for close at 1732071303448 (+1 ms)Writing region close event to WAL at 1732071303453 (+5 ms)Running coprocessor post-close hooks at 1732071303460 (+7 ms)Closed at 1732071303461 (+1 ms) 2024-11-20T02:55:03,462 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f04567280baab08e1176cbf031dac46b move to 1c82b83631a2,44699,1732071299007 record at close sequenceid=6 2024-11-20T02:55:03,465 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,465 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=CLOSED 2024-11-20T02:55:03,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829 because future has completed 2024-11-20T02:55:03,469 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T02:55:03,469 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T02:55:03,469 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,469 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,471 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T02:55:03,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T02:55:03,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:03,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T02:55:03,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071303469 2024-11-20T02:55:03,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,42601,1732071298829 in 182 msec 2024-11-20T02:55:03,473 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,474 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE; state=CLOSED, location=1c82b83631a2,44699,1732071299007; forceNewPlan=false, retain=false 2024-11-20T02:55:03,479 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:03,479 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071303471 2024-11-20T02:55:03,479 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,624 INFO [1c82b83631a2:40025 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T02:55:03,625 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=OPENING, regionLocation=1c82b83631a2,44699,1732071299007 2024-11-20T02:55:03,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE because future has completed 2024-11-20T02:55:03,630 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,44699,1732071299007}] 2024-11-20T02:55:03,785 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,785 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f04567280baab08e1176cbf031dac46b, NAME => 'testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:03,786 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732071301460 f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,786 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:03,786 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,786 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,788 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,788 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f04567280baab08e1176cbf031dac46b columnFamilyName family 2024-11-20T02:55:03,789 DEBUG [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:03,791 DEBUG [StoreFileOpener-f04567280baab08e1176cbf031dac46b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd, delay=973 ms 2024-11-20T02:55:03,792 DEBUG [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:03,792 INFO [StoreOpener-f04567280baab08e1176cbf031dac46b-1 {}] regionserver.HStore(327): Store=f04567280baab08e1176cbf031dac46b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:03,793 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,794 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,794 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,794 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,795 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,799 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,800 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f04567280baab08e1176cbf031dac46b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67336776, jitterRate=0.003396153450012207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:55:03,800 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,801 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f04567280baab08e1176cbf031dac46b: Running coprocessor pre-open hook at 1732071303787Writing region info on filesystem at 1732071303787Initializing all the Stores at 1732071303787Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071303787Cleaning up temporary data from old regions at 1732071303795 (+8 ms)Running coprocessor post-open hooks at 1732071303800 (+5 ms)Region opened successfully at 1732071303801 (+1 ms) 2024-11-20T02:55:03,801 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b., pid=9, masterSystemTime=1732071303782 2024-11-20T02:55:03,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f04567280baab08e1176cbf031dac46b, regionState=OPEN, openSeqNum=10, regionLocation=1c82b83631a2,44699,1732071299007 2024-11-20T02:55:03,805 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,805 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,44699,1732071299007 because future has completed 2024-11-20T02:55:03,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T02:55:03,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f04567280baab08e1176cbf031dac46b, server=1c82b83631a2,44699,1732071299007 in 180 msec 2024-11-20T02:55:03,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732071301460, region=f04567280baab08e1176cbf031dac46b, REOPEN/MOVE in 533 msec 2024-11-20T02:55:03,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T02:55:03,835 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:55:03,835 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:03,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:03,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:03,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T02:55:03,840 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T02:55:03,840 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=377653908, stopped=false 2024-11-20T02:55:03,841 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1c82b83631a2,40025,1732071297729 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:03,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:03,868 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:55:03,868 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:55:03,868 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:03,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:03,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,42601,1732071298829' ***** 2024-11-20T02:55:03,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T02:55:03,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:03,869 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,44699,1732071299007' ***** 2024-11-20T02:55:03,869 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T02:55:03,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:03,869 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:55:03,869 INFO [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T02:55:03,869 INFO [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T02:55:03,870 INFO [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence 2024-11-20T02:55:03,870 INFO [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/bucket.persistence 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T02:55:03,870 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T02:55:03,870 INFO [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T02:55:03,870 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T02:55:03,870 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:55:03,870 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T02:55:03,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:03,872 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:03,875 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:03,880 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1487): Thread RS:1;1c82b83631a2:44699 started persisting bucket cache to file 2024-11-20T02:55:03,880 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,880 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1487): Thread RS:0;1c82b83631a2:42601 started persisting bucket cache to file 2024-11-20T02:55:03,880 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,880 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:03,881 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:03,883 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:03,883 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:03,883 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071303880 2024-11-20T02:55:03,883 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071303880 2024-11-20T02:55:03,883 DEBUG [RS:1;1c82b83631a2:44699 {}] bucket.BucketCache(1509): Thread RS:1;1c82b83631a2:44699 finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,883 DEBUG [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1509): Thread RS:0;1c82b83631a2:42601 finished persisting bucket cache to file, renaming 2024-11-20T02:55:03,883 WARN [RS:0;1c82b83631a2:42601 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T02:55:03,883 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:55:03,883 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:55:03,884 INFO [RS:1;1c82b83631a2:44699 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:55:03,884 INFO [RS:1;1c82b83631a2:44699 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,42601,1732071298829 2024-11-20T02:55:03,884 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(3091): Received CLOSE for f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1c82b83631a2:42601. 2024-11-20T02:55:03,884 DEBUG [RS:0;1c82b83631a2:42601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:03,884 DEBUG [RS:0;1c82b83631a2:42601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:03,884 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:55:03,885 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:55:03,885 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:55:03,885 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T02:55:03,885 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,44699,1732071299007 2024-11-20T02:55:03,885 INFO [RS:1;1c82b83631a2:44699 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:03,885 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T02:55:03,885 INFO [RS:1;1c82b83631a2:44699 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1c82b83631a2:44699. 2024-11-20T02:55:03,885 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T02:55:03,885 DEBUG [RS:1;1c82b83631a2:44699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:03,885 DEBUG [RS:1;1c82b83631a2:44699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:03,885 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T02:55:03,885 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T02:55:03,885 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T02:55:03,885 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1325): Online Regions={f04567280baab08e1176cbf031dac46b=testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b.} 2024-11-20T02:55:03,886 DEBUG [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1351): Waiting on f04567280baab08e1176cbf031dac46b 2024-11-20T02:55:03,886 DEBUG [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f04567280baab08e1176cbf031dac46b, disabling compactions & flushes 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:55:03,886 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,886 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. after waiting 0 ms 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,886 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:55:03,886 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T02:55:03,887 DEBUG [StoreFileCloser-f04567280baab08e1176cbf031dac46b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/family/b70f76003ce646baaa79c386fda8bdfd 2024-11-20T02:55:03,891 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnRegionMove1732071301460/f04567280baab08e1176cbf031dac46b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T02:55:03,892 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,892 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f04567280baab08e1176cbf031dac46b: Waiting for close lock at 1732071303885Running coprocessor pre-close hooks at 1732071303886 (+1 ms)Disabling compacts and flushes for region at 1732071303886Disabling writes for close at 1732071303886Writing region close event to WAL at 1732071303887 (+1 ms)Running coprocessor post-close hooks at 1732071303892 (+5 ms)Closed at 1732071303892 2024-11-20T02:55:03,892 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b. 2024-11-20T02:55:03,897 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7bcb46c7dbe94b6a8502d757b3e2dea7 2024-11-20T02:55:03,913 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/7bcb46c7dbe94b6a8502d757b3e2dea7 is 205, key is testBlockEvictionOnRegionMove1732071301460,,1732071301474.f04567280baab08e1176cbf031dac46b./info:regioninfo/1732071303805/Put/seqid=0 2024-11-20T02:55:03,914 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/7bcb46c7dbe94b6a8502d757b3e2dea7 2024-11-20T02:55:03,922 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8205eb4b021b471a8e91ee5287b3d0cb 2024-11-20T02:55:03,934 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/8205eb4b021b471a8e91ee5287b3d0cb is 43, key is default/ns:d/1732071301217/Put/seqid=0 2024-11-20T02:55:03,935 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/8205eb4b021b471a8e91ee5287b3d0cb 2024-11-20T02:55:03,943 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7a4800a7fa38454b899b6299cc47592b 2024-11-20T02:55:03,958 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/7a4800a7fa38454b899b6299cc47592b is 78, key is testBlockEvictionOnRegionMove1732071301460/table:state/1732071301900/Put/seqid=0 2024-11-20T02:55:03,959 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/7a4800a7fa38454b899b6299cc47592b 2024-11-20T02:55:03,962 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/7bcb46c7dbe94b6a8502d757b3e2dea7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/7bcb46c7dbe94b6a8502d757b3e2dea7 2024-11-20T02:55:03,963 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/7bcb46c7dbe94b6a8502d757b3e2dea7, delay=911 ms 2024-11-20T02:55:03,965 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/7bcb46c7dbe94b6a8502d757b3e2dea7, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T02:55:03,965 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/8205eb4b021b471a8e91ee5287b3d0cb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/8205eb4b021b471a8e91ee5287b3d0cb 2024-11-20T02:55:03,967 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/8205eb4b021b471a8e91ee5287b3d0cb, delay=929 ms 2024-11-20T02:55:03,969 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/8205eb4b021b471a8e91ee5287b3d0cb, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T02:55:03,970 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/7a4800a7fa38454b899b6299cc47592b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/7a4800a7fa38454b899b6299cc47592b 2024-11-20T02:55:03,972 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/7a4800a7fa38454b899b6299cc47592b, delay=972 ms 2024-11-20T02:55:03,974 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/7a4800a7fa38454b899b6299cc47592b, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T02:55:03,975 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-20T02:55:03,975 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T02:55:03,976 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/7bcb46c7dbe94b6a8502d757b3e2dea7 2024-11-20T02:55:03,977 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/8205eb4b021b471a8e91ee5287b3d0cb 2024-11-20T02:55:03,977 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/7a4800a7fa38454b899b6299cc47592b 2024-11-20T02:55:03,982 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T02:55:03,982 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T02:55:03,983 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:03,983 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071303885Running coprocessor pre-close hooks at 1732071303886 (+1 ms)Disabling compacts and flushes for region at 1732071303886Disabling writes for close at 1732071303886Obtaining lock to block concurrent updates at 1732071303886Preparing flush snapshotting stores in 1588230740 at 1732071303886Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732071303887 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732071303887Flushing 1588230740/info: creating writer at 1732071303888 (+1 ms)Flushing 1588230740/info: appending metadata at 1732071303912 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732071303912Flushing 1588230740/ns: creating writer at 1732071303917 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732071303933 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732071303933Flushing 1588230740/table: creating writer at 1732071303937 (+4 ms)Flushing 1588230740/table: appending metadata at 1732071303957 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732071303958 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d2f2b7d: reopening flushed file at 1732071303961 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2abab607: reopening flushed file at 1732071303965 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54f72b3f: reopening flushed file at 1732071303970 (+5 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1732071303975 (+5 ms)Writing region close event to WAL at 1732071303978 (+3 ms)Running coprocessor post-close hooks at 1732071303982 (+4 ms)Closed at 1732071303983 (+1 ms) 2024-11-20T02:55:03,983 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:04,086 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,42601,1732071298829; all regions closed. 2024-11-20T02:55:04,086 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,44699,1732071299007; all regions closed. 2024-11-20T02:55:04,091 DEBUG [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:04,091 DEBUG [RS:1;1c82b83631a2:44699 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:04,091 INFO [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C42601%2C1732071298829.meta:.meta(num 1732071301073) 2024-11-20T02:55:04,091 INFO [RS:1;1c82b83631a2:44699 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C44699%2C1732071299007:(num 1732071300738) 2024-11-20T02:55:04,091 DEBUG [RS:1;1c82b83631a2:44699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:04,091 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:04,092 INFO [RS:1;1c82b83631a2:44699 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:04,092 INFO [RS:1;1c82b83631a2:44699 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:04,092 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:55:04,092 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:55:04,092 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:04,093 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:55:04,093 INFO [RS:1;1c82b83631a2:44699 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:04,093 INFO [RS:1;1c82b83631a2:44699 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44699 2024-11-20T02:55:04,095 DEBUG [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:04,095 INFO [RS:0;1c82b83631a2:42601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C42601%2C1732071298829:(num 1732071300738) 2024-11-20T02:55:04,095 DEBUG [RS:0;1c82b83631a2:42601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:04,095 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:04,095 INFO [RS:0;1c82b83631a2:42601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:04,095 INFO [RS:0;1c82b83631a2:42601 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:04,096 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:04,096 INFO [RS:0;1c82b83631a2:42601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:04,096 INFO [RS:0;1c82b83631a2:42601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42601 2024-11-20T02:55:04,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:04,122 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,44699,1732071299007 2024-11-20T02:55:04,123 INFO [RS:1;1c82b83631a2:44699 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:04,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,42601,1732071298829 2024-11-20T02:55:04,131 INFO [RS:0;1c82b83631a2:42601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:04,131 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,44699,1732071299007] 2024-11-20T02:55:04,147 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,44699,1732071299007 already deleted, retry=false 2024-11-20T02:55:04,148 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,44699,1732071299007 expired; onlineServers=1 2024-11-20T02:55:04,148 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,42601,1732071298829] 2024-11-20T02:55:04,159 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,42601,1732071298829 already deleted, retry=false 2024-11-20T02:55:04,159 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,42601,1732071298829 expired; onlineServers=0 2024-11-20T02:55:04,159 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1c82b83631a2,40025,1732071297729' ***** 2024-11-20T02:55:04,159 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T02:55:04,159 INFO [M:0;1c82b83631a2:40025 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:04,160 INFO [M:0;1c82b83631a2:40025 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:04,160 DEBUG [M:0;1c82b83631a2:40025 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T02:55:04,160 DEBUG [M:0;1c82b83631a2:40025 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T02:55:04,160 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T02:55:04,160 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071300270 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071300270,5,FailOnTimeoutGroup] 2024-11-20T02:55:04,160 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071300275 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071300275,5,FailOnTimeoutGroup] 2024-11-20T02:55:04,160 INFO [M:0;1c82b83631a2:40025 {}] hbase.ChoreService(370): Chore service for: master/1c82b83631a2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:04,160 INFO [M:0;1c82b83631a2:40025 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:04,161 DEBUG [M:0;1c82b83631a2:40025 {}] master.HMaster(1795): Stopping service threads 2024-11-20T02:55:04,161 INFO [M:0;1c82b83631a2:40025 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T02:55:04,161 INFO [M:0;1c82b83631a2:40025 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:55:04,162 INFO [M:0;1c82b83631a2:40025 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T02:55:04,162 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T02:55:04,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T02:55:04,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,168 DEBUG [M:0;1c82b83631a2:40025 {}] zookeeper.ZKUtil(347): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T02:55:04,168 WARN [M:0;1c82b83631a2:40025 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T02:55:04,168 INFO [M:0;1c82b83631a2:40025 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.lastflushedseqids 2024-11-20T02:55:04,180 INFO [M:0;1c82b83631a2:40025 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T02:55:04,181 INFO [M:0;1c82b83631a2:40025 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T02:55:04,181 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:55:04,181 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,181 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,181 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:55:04,181 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,181 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-20T02:55:04,190 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d5b188b3542410f8e664349b2582317 2024-11-20T02:55:04,204 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b188b3542410f8e664349b2582317 is 82, key is hbase:meta,,1/info:regioninfo/1732071301131/Put/seqid=0 2024-11-20T02:55:04,205 INFO [M:0;1c82b83631a2:40025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b188b3542410f8e664349b2582317 2024-11-20T02:55:04,213 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6640b333dc64ac6a70a83c6d3ed1644 2024-11-20T02:55:04,227 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f6640b333dc64ac6a70a83c6d3ed1644 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732071301906/Put/seqid=0 2024-11-20T02:55:04,228 INFO [M:0;1c82b83631a2:40025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f6640b333dc64ac6a70a83c6d3ed1644 2024-11-20T02:55:04,236 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cbc7264fbe124f98a8a6271ef1bf587e 2024-11-20T02:55:04,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44699-0x101577424340002, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,240 INFO [RS:1;1c82b83631a2:44699 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:04,240 INFO [RS:1;1c82b83631a2:44699 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,44699,1732071299007; zookeeper connection closed. 2024-11-20T02:55:04,240 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@591cbe05 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@591cbe05 2024-11-20T02:55:04,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,248 INFO [RS:0;1c82b83631a2:42601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:04,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42601-0x101577424340001, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,248 INFO [RS:0;1c82b83631a2:42601 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,42601,1732071298829; zookeeper connection closed. 2024-11-20T02:55:04,248 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f95a201 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f95a201 2024-11-20T02:55:04,248 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T02:55:04,249 DEBUG [M:0;1c82b83631a2:40025 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbc7264fbe124f98a8a6271ef1bf587e is 69, key is 1c82b83631a2,42601,1732071298829/rs:state/1732071300398/Put/seqid=0 2024-11-20T02:55:04,249 INFO [M:0;1c82b83631a2:40025 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbc7264fbe124f98a8a6271ef1bf587e 2024-11-20T02:55:04,251 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2d5b188b3542410f8e664349b2582317 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d5b188b3542410f8e664349b2582317 2024-11-20T02:55:04,253 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2d5b188b3542410f8e664349b2582317, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T02:55:04,254 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f6640b333dc64ac6a70a83c6d3ed1644 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f6640b333dc64ac6a70a83c6d3ed1644 2024-11-20T02:55:04,256 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f6640b333dc64ac6a70a83c6d3ed1644, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T02:55:04,256 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cbc7264fbe124f98a8a6271ef1bf587e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbc7264fbe124f98a8a6271ef1bf587e 2024-11-20T02:55:04,258 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cbc7264fbe124f98a8a6271ef1bf587e, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T02:55:04,259 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false 2024-11-20T02:55:04,261 INFO [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,261 DEBUG [M:0;1c82b83631a2:40025 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071304181Disabling compacts and flushes for region at 1732071304181Disabling writes for close at 1732071304181Obtaining lock to block concurrent updates at 1732071304181Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732071304181Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732071304182 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732071304182Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732071304182Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732071304203 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732071304203Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732071304207 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732071304226 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732071304226Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732071304230 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732071304248 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732071304248Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41ade2c5: reopening flushed file at 1732071304251 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f802495: reopening flushed file at 1732071304253 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8c1f8f4: reopening flushed file at 1732071304256 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false at 1732071304259 (+3 ms)Writing region close event to WAL at 1732071304261 (+2 ms)Closed at 1732071304261 2024-11-20T02:55:04,263 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:04,263 INFO [M:0;1c82b83631a2:40025 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T02:55:04,263 INFO [M:0;1c82b83631a2:40025 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40025 2024-11-20T02:55:04,263 INFO [M:0;1c82b83631a2:40025 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:04,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,393 INFO [M:0;1c82b83631a2:40025 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:04,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40025-0x101577424340000, quorum=127.0.0.1:54683, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:04,403 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T02:55:04,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T02:55:04,423 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732071302252 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732071303963 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732071303791 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/1c82b83631a2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: master/1c82b83631a2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/1c82b83631a2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732071303967 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/1c82b83631a2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=375 (was 356) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6916 (was 7559) 2024-11-20T02:55:04,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=375, ProcessCount=11, AvailableMemoryMB=6916 2024-11-20T02:55:04,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/hadoop-log-dir so I do NOT create it in target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9 2024-11-20T02:55:04,438 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/hadoop-tmp-dir so I do NOT create it in target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9 2024-11-20T02:55:04,438 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,441 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/cluster_17b7b513-278d-8d8b-13c4-36fc5a6b0d70, deleteOnExit=true 2024-11-20T02:55:04,445 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/cluster_17b7b513-278d-8d8b-13c4-36fc5a6b0d70/zookeeper_0, clientPort=50458, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/cluster_17b7b513-278d-8d8b-13c4-36fc5a6b0d70/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/cluster_17b7b513-278d-8d8b-13c4-36fc5a6b0d70/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T02:55:04,447 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50458 2024-11-20T02:55:04,460 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 with version=8 2024-11-20T02:55:04,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase-staging 2024-11-20T02:55:04,463 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:55:04,463 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T02:55:04,464 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:55:04,465 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41191 2024-11-20T02:55:04,467 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41191 connecting to ZooKeeper ensemble=127.0.0.1:50458 2024-11-20T02:55:04,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:411910x0, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:55:04,505 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41191-0x10157743fc00000 connected 2024-11-20T02:55:04,596 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,597 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:04,601 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98, hbase.cluster.distributed=false 2024-11-20T02:55:04,607 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:55:04,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-20T02:55:04,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41191 2024-11-20T02:55:04,609 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41191 2024-11-20T02:55:04,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-20T02:55:04,610 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41191 2024-11-20T02:55:04,625 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:55:04,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:55:04,626 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:55:04,627 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38797 2024-11-20T02:55:04,629 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38797 connecting to ZooKeeper ensemble=127.0.0.1:50458 2024-11-20T02:55:04,629 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,629 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387970x0, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:55:04,641 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38797-0x10157743fc00001 connected 2024-11-20T02:55:04,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:04,642 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:55:04,642 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache 2024-11-20T02:55:04,642 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T02:55:04,643 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T02:55:04,643 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T02:55:04,643 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T02:55:04,643 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T02:55:04,643 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T02:55:04,644 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:55:04,644 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T02:55:04,644 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence is complete. 2024-11-20T02:55:04,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T02:55:04,646 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:55:04,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38797 2024-11-20T02:55:04,647 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38797 2024-11-20T02:55:04,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38797 2024-11-20T02:55:04,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38797 2024-11-20T02:55:04,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38797 2024-11-20T02:55:04,668 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:55:04,668 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:55:04,669 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34645 2024-11-20T02:55:04,671 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34645 connecting to ZooKeeper ensemble=127.0.0.1:50458 2024-11-20T02:55:04,671 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,671 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:346450x0, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:55:04,691 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:346450x0, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:04,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34645-0x10157743fc00002 connected 2024-11-20T02:55:04,692 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:55:04,693 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache 2024-11-20T02:55:04,693 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T02:55:04,693 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T02:55:04,693 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T02:55:04,693 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T02:55:04,693 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T02:55:04,693 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T02:55:04,693 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T02:55:04,693 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:55:04,693 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence is complete. 2024-11-20T02:55:04,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T02:55:04,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:55:04,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34645 2024-11-20T02:55:04,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34645 2024-11-20T02:55:04,699 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34645 2024-11-20T02:55:04,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34645 2024-11-20T02:55:04,703 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34645 2024-11-20T02:55:04,715 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1c82b83631a2:41191 2024-11-20T02:55:04,715 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1c82b83631a2,41191,1732071304462 2024-11-20T02:55:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,725 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1c82b83631a2,41191,1732071304462 2024-11-20T02:55:04,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T02:55:04,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T02:55:04,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,733 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T02:55:04,733 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1c82b83631a2,41191,1732071304462 from backup master directory 2024-11-20T02:55:04,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1c82b83631a2,41191,1732071304462 2024-11-20T02:55:04,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T02:55:04,741 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:04,741 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1c82b83631a2,41191,1732071304462 2024-11-20T02:55:04,742 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase.id] with ID: ccf8c293-42d8-41f2-bb0e-8cacd6c03b29 2024-11-20T02:55:04,742 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.tmp/hbase.id 2024-11-20T02:55:04,745 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/hbase.id] 2024-11-20T02:55:04,781 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:04,781 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T02:55:04,786 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 5ms. 2024-11-20T02:55:04,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,804 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:55:04,805 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T02:55:04,805 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:04,828 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store 2024-11-20T02:55:04,836 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:04,837 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:55:04,837 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,837 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,837 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:55:04,837 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,837 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:04,837 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071304837Disabling compacts and flushes for region at 1732071304837Disabling writes for close at 1732071304837Writing region close event to WAL at 1732071304837Closed at 1732071304837 2024-11-20T02:55:04,838 WARN [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/.initializing 2024-11-20T02:55:04,838 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,41191,1732071304462 2024-11-20T02:55:04,844 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:04,848 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1c82b83631a2%2C41191%2C1732071304462, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,41191,1732071304462, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/oldWALs, maxLogs=10 2024-11-20T02:55:04,869 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/WALs/1c82b83631a2,41191,1732071304462/1c82b83631a2%2C41191%2C1732071304462.1732071304849 2024-11-20T02:55:04,869 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:04,869 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:04,870 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:04,870 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,870 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,871 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,875 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T02:55:04,876 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:04,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:04,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T02:55:04,880 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:04,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:04,880 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T02:55:04,884 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:04,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:04,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T02:55:04,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:04,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:04,889 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,890 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,890 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,890 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,890 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,891 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:55:04,891 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T02:55:04,899 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:55:04,900 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60744410, jitterRate=-0.09483775496482849}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:55:04,901 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732071304870Initializing all the Stores at 1732071304870Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071304871 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071304871Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071304871Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071304871Cleaning up temporary data from old regions at 1732071304890 (+19 ms)Region opened successfully at 1732071304901 (+11 ms) 2024-11-20T02:55:04,903 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T02:55:04,907 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb19fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:04,908 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T02:55:04,908 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T02:55:04,908 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T02:55:04,909 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T02:55:04,909 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T02:55:04,909 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T02:55:04,909 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T02:55:04,912 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T02:55:04,913 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T02:55:04,929 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T02:55:04,929 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T02:55:04,930 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T02:55:04,940 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T02:55:04,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T02:55:04,942 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T02:55:04,949 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T02:55:04,950 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T02:55:04,957 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T02:55:04,961 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T02:55:04,971 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T02:55:04,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:04,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:04,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:04,982 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,983 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1c82b83631a2,41191,1732071304462, sessionid=0x10157743fc00000, setting cluster-up flag (Was=false) 2024-11-20T02:55:04,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:04,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,049 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T02:55:05,051 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,41191,1732071304462 2024-11-20T02:55:05,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,099 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T02:55:05,101 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1c82b83631a2,41191,1732071304462 2024-11-20T02:55:05,101 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T02:55:05,103 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:05,104 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T02:55:05,104 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T02:55:05,104 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1c82b83631a2,41191,1732071304462 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T02:55:05,105 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(746): ClusterId : ccf8c293-42d8-41f2-bb0e-8cacd6c03b29 2024-11-20T02:55:05,106 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1c82b83631a2:0, corePoolSize=5, maxPoolSize=5 2024-11-20T02:55:05,116 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(746): ClusterId : ccf8c293-42d8-41f2-bb0e-8cacd6c03b29 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1c82b83631a2:0, corePoolSize=10, maxPoolSize=10 2024-11-20T02:55:05,116 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:05,116 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,117 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:55:05,117 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:55:05,118 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:05,119 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T02:55:05,119 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,120 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T02:55:05,130 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732071335130 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T02:55:05,131 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,132 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T02:55:05,132 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T02:55:05,132 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T02:55:05,133 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T02:55:05,133 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T02:55:05,133 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:55:05,133 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:55:05,134 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:55:05,134 DEBUG [RS:0;1c82b83631a2:38797 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f31f62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:05,136 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071305133,5,FailOnTimeoutGroup] 2024-11-20T02:55:05,139 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071305136,5,FailOnTimeoutGroup] 2024-11-20T02:55:05,139 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,139 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T02:55:05,139 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,139 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,145 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T02:55:05,146 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:05,147 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:55:05,148 DEBUG [RS:1;1c82b83631a2:34645 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0d295c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:05,153 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1c82b83631a2:38797 2024-11-20T02:55:05,153 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:55:05,153 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:55:05,153 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:55:05,155 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,41191,1732071304462 with port=38797, startcode=1732071304625 2024-11-20T02:55:05,155 DEBUG [RS:0;1c82b83631a2:38797 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:55:05,155 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:05,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T02:55:05,159 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37035, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:55:05,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,160 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,162 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:05,163 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T02:55:05,163 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:55:05,165 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;1c82b83631a2:34645 2024-11-20T02:55:05,165 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:55:05,165 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:55:05,165 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:55:05,168 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:55:05,168 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,168 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,169 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T02:55:05,172 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:55:05,173 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,173 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T02:55:05,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:05,174 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,41191,1732071304462 with port=34645, startcode=1732071304667 2024-11-20T02:55:05,174 DEBUG [RS:1;1c82b83631a2:34645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:55:05,174 DEBUG [RS:0;1c82b83631a2:38797 {}] zookeeper.ZKUtil(111): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,174 WARN [RS:0;1c82b83631a2:38797 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:05,175 INFO [RS:0;1c82b83631a2:38797 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:05,175 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,38797,1732071304625] 2024-11-20T02:55:05,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:55:05,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,178 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41799, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:55:05,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T02:55:05,179 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,179 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,181 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:55:05,181 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:05,181 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T02:55:05,181 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:55:05,183 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:55:05,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:55:05,184 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,184 INFO [RS:0;1c82b83631a2:38797 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:55:05,184 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,184 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:55:05,184 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:55:05,184 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:55:05,185 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,185 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:05,185 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:05,186 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:55:05,186 DEBUG [RS:0;1c82b83631a2:38797 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:05,189 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,38797,1732071304625-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:05,192 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:55:05,193 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67374423, jitterRate=0.00395713746547699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:55:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732071305156Initializing all the Stores at 1732071305156Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305156Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305158 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071305158Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305158Cleaning up temporary data from old regions at 1732071305185 (+27 ms)Region opened successfully at 1732071305193 (+8 ms) 2024-11-20T02:55:05,193 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:55:05,194 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:55:05,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:55:05,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:55:05,194 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:55:05,195 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:05,195 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071305193Disabling compacts and flushes for region at 1732071305193Disabling writes for close at 1732071305194 (+1 ms)Writing region close event to WAL at 1732071305195 (+1 ms)Closed at 1732071305195 2024-11-20T02:55:05,196 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:05,196 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T02:55:05,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T02:55:05,198 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:55:05,199 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T02:55:05,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:05,207 DEBUG [RS:1;1c82b83631a2:34645 {}] zookeeper.ZKUtil(111): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,207 WARN [RS:1;1c82b83631a2:34645 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:05,207 INFO [RS:1;1c82b83631a2:34645 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:05,207 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,208 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,34645,1732071304667] 2024-11-20T02:55:05,214 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:55:05,215 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:55:05,216 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:55:05,216 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,38797,1732071304625-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,216 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,216 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.Replication(171): 1c82b83631a2,38797,1732071304625 started 2024-11-20T02:55:05,216 INFO [RS:1;1c82b83631a2:34645 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:55:05,217 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,217 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:55:05,218 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:55:05,218 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,218 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,218 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,218 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,218 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,218 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:05,219 DEBUG [RS:1;1c82b83631a2:34645 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:05,220 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,221 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,221 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,221 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,221 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,221 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,34645,1732071304667-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:05,238 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,238 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,38797,1732071304625, RpcServer on 1c82b83631a2/172.17.0.2:38797, sessionid=0x10157743fc00001 2024-11-20T02:55:05,238 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:55:05,238 DEBUG [RS:0;1c82b83631a2:38797 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,238 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,38797,1732071304625' 2024-11-20T02:55:05,238 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:55:05,239 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:55:05,239 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:55:05,240 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:55:05,240 DEBUG [RS:0;1c82b83631a2:38797 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,240 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,38797,1732071304625' 2024-11-20T02:55:05,240 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:55:05,241 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:55:05,241 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,34645,1732071304667-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,241 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,241 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.Replication(171): 1c82b83631a2,34645,1732071304667 started 2024-11-20T02:55:05,243 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:55:05,243 DEBUG [RS:0;1c82b83631a2:38797 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:55:05,244 INFO [RS:0;1c82b83631a2:38797 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:55:05,244 INFO [RS:0;1c82b83631a2:38797 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:55:05,260 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,260 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,34645,1732071304667, RpcServer on 1c82b83631a2/172.17.0.2:34645, sessionid=0x10157743fc00002 2024-11-20T02:55:05,260 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:55:05,260 DEBUG [RS:1;1c82b83631a2:34645 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,260 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,34645,1732071304667' 2024-11-20T02:55:05,260 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,34645,1732071304667 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,34645,1732071304667' 2024-11-20T02:55:05,261 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:55:05,262 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:55:05,262 DEBUG [RS:1;1c82b83631a2:34645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:55:05,262 INFO [RS:1;1c82b83631a2:34645 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:55:05,262 INFO [RS:1;1c82b83631a2:34645 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:55:05,344 INFO [RS:0;1c82b83631a2:38797 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:05,345 INFO [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C38797%2C1732071304625, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,38797,1732071304625, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:05,349 WARN [1c82b83631a2:41191 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T02:55:05,363 INFO [RS:1;1c82b83631a2:34645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:05,364 INFO [RS:1;1c82b83631a2:34645 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C34645%2C1732071304667, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:05,374 INFO [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,38797,1732071304625/1c82b83631a2%2C38797%2C1732071304625.1732071305348 2024-11-20T02:55:05,375 DEBUG [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:05,384 INFO [RS:1;1c82b83631a2:34645 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667/1c82b83631a2%2C34645%2C1732071304667.1732071305365 2024-11-20T02:55:05,385 DEBUG [RS:1;1c82b83631a2:34645 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:05,600 DEBUG [1c82b83631a2:41191 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T02:55:05,600 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T02:55:05,601 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T02:55:05,601 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T02:55:05,601 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T02:55:05,601 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T02:55:05,601 INFO [1c82b83631a2:41191 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T02:55:05,602 INFO [1c82b83631a2:41191 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T02:55:05,602 DEBUG [1c82b83631a2:41191 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T02:55:05,602 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,603 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,38797,1732071304625, state=OPENING 2024-11-20T02:55:05,624 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T02:55:05,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:05,680 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T02:55:05,680 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,38797,1732071304625}] 2024-11-20T02:55:05,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,681 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,834 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T02:55:05,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42979, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T02:55:05,842 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T02:55:05,842 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:05,843 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T02:55:05,845 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C38797%2C1732071304625.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,38797,1732071304625, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:05,870 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,38797,1732071304625/1c82b83631a2%2C38797%2C1732071304625.meta.1732071305847.meta 2024-11-20T02:55:05,870 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:05,870 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:05,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T02:55:05,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T02:55:05,871 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T02:55:05,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T02:55:05,871 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:05,872 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T02:55:05,872 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T02:55:05,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T02:55:05,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T02:55:05,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T02:55:05,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T02:55:05,875 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T02:55:05,875 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T02:55:05,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T02:55:05,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T02:55:05,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:05,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T02:55:05,877 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T02:55:05,878 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:05,878 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740 2024-11-20T02:55:05,879 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T02:55:05,879 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T02:55:05,879 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T02:55:05,880 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T02:55:05,880 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69497157, jitterRate=0.035588338971138}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T02:55:05,881 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T02:55:05,882 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732071305872Writing region info on filesystem at 1732071305872Initializing all the Stores at 1732071305872Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305872Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305872Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071305872Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732071305872Cleaning up temporary data from old regions at 1732071305879 (+7 ms)Running coprocessor post-open hooks at 1732071305881 (+2 ms)Region opened successfully at 1732071305881 2024-11-20T02:55:05,882 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732071305833 2024-11-20T02:55:05,885 DEBUG [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T02:55:05,886 INFO [RS_OPEN_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T02:55:05,886 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,886 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1c82b83631a2,38797,1732071304625, state=OPEN 2024-11-20T02:55:05,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:05,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:05,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T02:55:05,916 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:05,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,916 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T02:55:05,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T02:55:05,918 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1c82b83631a2,38797,1732071304625 in 236 msec 2024-11-20T02:55:05,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T02:55:05,920 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 722 msec 2024-11-20T02:55:05,920 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T02:55:05,920 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T02:55:05,923 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:55:05,923 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,38797,1732071304625, seqNum=-1] 2024-11-20T02:55:05,923 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:55:05,925 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:55:05,931 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 827 msec 2024-11-20T02:55:05,931 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732071305931, completionTime=-1 2024-11-20T02:55:05,931 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T02:55:05,931 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T02:55:05,934 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T02:55:05,934 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732071365934 2024-11-20T02:55:05,934 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732071425934 2024-11-20T02:55:05,934 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-20T02:55:05,934 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T02:55:05,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1c82b83631a2:41191, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,935 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,936 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:05,938 DEBUG [master/1c82b83631a2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.200sec 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:05,941 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T02:55:05,942 DEBUG [master/1c82b83631a2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T02:55:05,942 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T02:55:05,942 INFO [master/1c82b83631a2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,41191,1732071304462-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:06,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310468de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:06,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1c82b83631a2,41191,-1 for getting cluster id 2024-11-20T02:55:06,010 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T02:55:06,013 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ccf8c293-42d8-41f2-bb0e-8cacd6c03b29' 2024-11-20T02:55:06,014 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T02:55:06,015 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ccf8c293-42d8-41f2-bb0e-8cacd6c03b29" 2024-11-20T02:55:06,015 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20550dca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:06,016 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1c82b83631a2,41191,-1] 2024-11-20T02:55:06,016 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T02:55:06,017 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:06,019 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T02:55:06,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b67efd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T02:55:06,021 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T02:55:06,023 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1c82b83631a2,38797,1732071304625, seqNum=-1] 2024-11-20T02:55:06,024 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:55:06,027 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:55:06,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1c82b83631a2,41191,1732071304462 2024-11-20T02:55:06,030 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T02:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1c82b83631a2,41191,1732071304462 2024-11-20T02:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ae1ad91 2024-11-20T02:55:06,032 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T02:55:06,036 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T02:55:06,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732071306029', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T02:55:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 2024-11-20T02:55:06,040 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T02:55:06,040 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:06,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732071306029" procId is: 4 2024-11-20T02:55:06,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T02:55:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:06,054 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a13e31ee199df694a93917b8db953fb, NAME => 'testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732071306029', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:06,067 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:06,068 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1722): Closing 1a13e31ee199df694a93917b8db953fb, disabling compactions & flushes 2024-11-20T02:55:06,068 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,068 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,068 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. after waiting 0 ms 2024-11-20T02:55:06,068 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,068 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,068 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732071306029-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a13e31ee199df694a93917b8db953fb: Waiting for close lock at 1732071306068Disabling compacts and flushes for region at 1732071306068Disabling writes for close at 1732071306068Writing region close event to WAL at 1732071306068Closed at 1732071306068 2024-11-20T02:55:06,070 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T02:55:06,071 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732071306070"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732071306070"}]},"ts":"1732071306070"} 2024-11-20T02:55:06,075 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T02:55:06,076 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T02:55:06,077 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732071306029","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071306076"}]},"ts":"1732071306076"} 2024-11-20T02:55:06,081 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732071306029, state=ENABLING in hbase:meta 2024-11-20T02:55:06,081 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {1c82b83631a2=0} racks are {/default-rack=0} 2024-11-20T02:55:06,082 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T02:55:06,082 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T02:55:06,082 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T02:55:06,082 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T02:55:06,082 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T02:55:06,082 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T02:55:06,082 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T02:55:06,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN}] 2024-11-20T02:55:06,084 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN 2024-11-20T02:55:06,086 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN; state=OFFLINE, location=1c82b83631a2,34645,1732071304667; forceNewPlan=false, retain=false 2024-11-20T02:55:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:06,237 INFO [1c82b83631a2:41191 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T02:55:06,237 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a13e31ee199df694a93917b8db953fb, regionState=OPENING, regionLocation=1c82b83631a2,34645,1732071304667 2024-11-20T02:55:06,241 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN because future has completed 2024-11-20T02:55:06,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,34645,1732071304667}] 2024-11-20T02:55:06,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:06,395 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T02:55:06,397 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33767, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T02:55:06,400 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,401 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a13e31ee199df694a93917b8db953fb, NAME => 'testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:06,401 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732071306029 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,402 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:06,402 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,402 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,403 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,410 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a13e31ee199df694a93917b8db953fb columnFamilyName family 2024-11-20T02:55:06,410 DEBUG [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:06,410 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] regionserver.HStore(327): Store=1a13e31ee199df694a93917b8db953fb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:06,411 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,412 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,412 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,412 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,412 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,413 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,419 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T02:55:06,419 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1a13e31ee199df694a93917b8db953fb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71106732, jitterRate=0.05957287549972534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:55:06,419 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:06,420 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1a13e31ee199df694a93917b8db953fb: Running coprocessor pre-open hook at 1732071306402Writing region info on filesystem at 1732071306402Initializing all the Stores at 1732071306403 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071306403Cleaning up temporary data from old regions at 1732071306412 (+9 ms)Running coprocessor post-open hooks at 1732071306419 (+7 ms)Region opened successfully at 1732071306420 (+1 ms) 2024-11-20T02:55:06,420 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb., pid=6, masterSystemTime=1732071306395 2024-11-20T02:55:06,423 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,423 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:06,423 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1a13e31ee199df694a93917b8db953fb, regionState=OPEN, openSeqNum=2, regionLocation=1c82b83631a2,34645,1732071304667 2024-11-20T02:55:06,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,34645,1732071304667 because future has completed 2024-11-20T02:55:06,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T02:55:06,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,34645,1732071304667 in 185 msec 2024-11-20T02:55:06,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T02:55:06,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN in 345 msec 2024-11-20T02:55:06,431 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T02:55:06,431 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732071306029","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732071306431"}]},"ts":"1732071306431"} 2024-11-20T02:55:06,433 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732071306029, state=ENABLED in hbase:meta 2024-11-20T02:55:06,434 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T02:55:06,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732071306029 in 397 msec 2024-11-20T02:55:06,607 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T02:55:06,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41191 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T02:55:06,665 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732071306029 completed 2024-11-20T02:55:06,665 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732071306029 get assigned. Timeout = 60000ms 2024-11-20T02:55:06,665 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:55:06,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732071306029 assigned to meta. Checking AM states. 2024-11-20T02:55:06,668 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T02:55:06,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732071306029 assigned. 2024-11-20T02:55:06,672 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732071306029', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb., hostname=1c82b83631a2,34645,1732071304667, seqNum=2] 2024-11-20T02:55:06,672 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T02:55:06,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T02:55:06,679 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1a13e31ee199df694a93917b8db953fb 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T02:55:06,687 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:06,704 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/.tmp/family/79be5daf5f31443080fca040f08b682c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T02:55:06,705 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/.tmp/family/79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:06,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/.tmp/family/79be5daf5f31443080fca040f08b682c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:06,709 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, delay=986 ms 2024-11-20T02:55:06,710 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T02:55:06,711 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1a13e31ee199df694a93917b8db953fb in 32ms, sequenceid=6, compaction requested=false 2024-11-20T02:55:06,711 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732071306029' 2024-11-20T02:55:06,712 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1a13e31ee199df694a93917b8db953fb: 2024-11-20T02:55:07,696 DEBUG [hfile-prefetch-1732071303791 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 79be5daf5f31443080fca040f08b682c_0. 2024-11-20T02:55:07,697 DEBUG [hfile-prefetch-1732071303791 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 79be5daf5f31443080fca040f08b682c_117. 2024-11-20T02:55:07,698 DEBUG [hfile-prefetch-1732071303791 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, with total blocks 2, and data blocks 1 2024-11-20T02:55:07,698 DEBUG [hfile-prefetch-1732071303791 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T02:55:07,698 DEBUG [hfile-prefetch-1732071303791 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T02:55:07,698 DEBUG [hfile-prefetch-1732071303791 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c. Will sleep 100ms and try the verification again. 2024-11-20T02:55:07,712 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;1c82b83631a2:34645,5,FailOnTimeoutGroup] 2024-11-20T02:55:07,712 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,34645,1732071304667' ***** 2024-11-20T02:55:07,712 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-20T02:55:07,713 INFO [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T02:55:07,724 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:55:07,724 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:07,740 INFO [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence 2024-11-20T02:55:07,740 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T02:55:07,740 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T02:55:07,740 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T02:55:07,740 INFO [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T02:55:07,741 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T02:55:07,746 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T02:55:07,746 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T02:55:07,746 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:07,746 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:07,751 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1487): Thread RS:1;1c82b83631a2:34645 started persisting bucket cache to file 2024-11-20T02:55:07,751 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:07,751 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:07,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:07,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071307746 2024-11-20T02:55:07,756 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T02:55:07,757 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T02:55:07,757 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732071307751 2024-11-20T02:55:07,757 DEBUG [RS:1;1c82b83631a2:34645 {}] bucket.BucketCache(1509): Thread RS:1;1c82b83631a2:34645 finished persisting bucket cache to file, renaming 2024-11-20T02:55:07,757 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,34645,1732071304667 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;1c82b83631a2:34645. 2024-11-20T02:55:07,758 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T02:55:07,758 DEBUG [RS:1;1c82b83631a2:34645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:07,758 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a13e31ee199df694a93917b8db953fb, disabling compactions & flushes 2024-11-20T02:55:07,758 DEBUG [RS:1;1c82b83631a2:34645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:07,758 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:07,758 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:07,758 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T02:55:07,758 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. after waiting 0 ms 2024-11-20T02:55:07,758 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1325): Online Regions={1a13e31ee199df694a93917b8db953fb=testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.} 2024-11-20T02:55:07,759 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:07,759 DEBUG [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1351): Waiting on 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:07,759 DEBUG [StoreFileCloser-1a13e31ee199df694a93917b8db953fb-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:07,759 WARN [hfile-prefetch-1732071303791 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T02:55:07,760 DEBUG [hfile-prefetch-1732071303791 {}] hfile.PrefetchExecutor(121): Prefetch completed for 79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:07,763 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T02:55:07,763 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:07,764 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a13e31ee199df694a93917b8db953fb: Waiting for close lock at 1732071307758Running coprocessor pre-close hooks at 1732071307758Disabling compacts and flushes for region at 1732071307758Disabling writes for close at 1732071307758Writing region close event to WAL at 1732071307759 (+1 ms)Running coprocessor post-close hooks at 1732071307763 (+4 ms)Closed at 1732071307763 2024-11-20T02:55:07,764 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:07,959 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,34645,1732071304667; all regions closed. 2024-11-20T02:55:07,965 DEBUG [RS:1;1c82b83631a2:34645 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:07,965 INFO [RS:1;1c82b83631a2:34645 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C34645%2C1732071304667:(num 1732071305365) 2024-11-20T02:55:07,965 DEBUG [RS:1;1c82b83631a2:34645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:07,965 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:07,966 INFO [RS:1;1c82b83631a2:34645 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:07,966 INFO [RS:1;1c82b83631a2:34645 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:07,967 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:55:07,967 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:07,967 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:55:07,967 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:55:07,967 INFO [RS:1;1c82b83631a2:34645 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:07,968 INFO [RS:1;1c82b83631a2:34645 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34645 2024-11-20T02:55:07,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,34645,1732071304667 2024-11-20T02:55:07,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:07,987 INFO [RS:1;1c82b83631a2:34645 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:07,998 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,34645,1732071304667] 2024-11-20T02:55:08,006 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,34645,1732071304667 already deleted, retry=false 2024-11-20T02:55:08,007 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 1c82b83631a2,34645,1732071304667 on 1c82b83631a2,41191,1732071304462 2024-11-20T02:55:08,010 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 1c82b83631a2,34645,1732071304667, splitWal=true, meta=false 2024-11-20T02:55:08,012 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 1c82b83631a2,34645,1732071304667, splitWal=true, meta=false 2024-11-20T02:55:08,012 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 1c82b83631a2,34645,1732071304667 (carryingMeta=false) 1c82b83631a2,34645,1732071304667/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@399cc4df[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T02:55:08,013 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 1c82b83631a2,34645,1732071304667 had 1 regions 2024-11-20T02:55:08,014 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 1c82b83631a2,34645,1732071304667, splitWal=true, meta=false, isMeta: false 2024-11-20T02:55:08,015 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667-splitting 2024-11-20T02:55:08,015 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667-splitting dir is empty, no logs to split. 2024-11-20T02:55:08,015 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1c82b83631a2,34645,1732071304667 WAL count=0, meta=false 2024-11-20T02:55:08,018 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667-splitting dir is empty, no logs to split. 2024-11-20T02:55:08,018 INFO [PEWorker-2 {}] master.SplitWALManager(105): 1c82b83631a2,34645,1732071304667 WAL count=0, meta=false 2024-11-20T02:55:08,018 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 1c82b83631a2,34645,1732071304667 WAL splitting is done? wals=0, meta=false 2024-11-20T02:55:08,019 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 1c82b83631a2,34645,1732071304667 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,34645,1732071304667-splitting does not exist 2024-11-20T02:55:08,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN}] 2024-11-20T02:55:08,023 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN 2024-11-20T02:55:08,024 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T02:55:08,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:08,099 INFO [RS:1;1c82b83631a2:34645 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:08,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34645-0x10157743fc00002, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:08,099 INFO [RS:1;1c82b83631a2:34645 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,34645,1732071304667; zookeeper connection closed. 2024-11-20T02:55:08,099 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6527a6f0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6527a6f0 2024-11-20T02:55:08,175 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1a13e31ee199df694a93917b8db953fb, regionState=OPENING, regionLocation=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:08,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN because future has completed 2024-11-20T02:55:08,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,38797,1732071304625}] 2024-11-20T02:55:08,231 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1c82b83631a2:0 server-side Connection retries=45 2024-11-20T02:55:08,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:08,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:08,231 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T02:55:08,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T02:55:08,231 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T02:55:08,232 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T02:55:08,232 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T02:55:08,232 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46017 2024-11-20T02:55:08,234 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46017 connecting to ZooKeeper ensemble=127.0.0.1:50458 2024-11-20T02:55:08,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:08,234 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:08,262 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T02:55:08,263 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T02:55:08,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460170x0, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T02:55:08,266 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T02:55:08,266 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T02:55:08,266 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T02:55:08,266 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46017-0x10157743fc00003 connected 2024-11-20T02:55:08,266 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T02:55:08,267 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T02:55:08,267 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T02:55:08,268 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T02:55:08,269 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache 2024-11-20T02:55:08,269 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T02:55:08,269 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T02:55:08,269 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T02:55:08,269 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T02:55:08,270 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T02:55:08,270 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T02:55:08,271 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T02:55:08,271 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T02:55:08,272 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-20T02:55:08,273 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T02:55:08,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46017 2024-11-20T02:55:08,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46017 2024-11-20T02:55:08,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46017 2024-11-20T02:55:08,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46017 2024-11-20T02:55:08,279 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46017 2024-11-20T02:55:08,279 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T02:55:08,280 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T02:55:08,280 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-20T02:55:08,280 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence is complete. 2024-11-20T02:55:08,282 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(746): ClusterId : ccf8c293-42d8-41f2-bb0e-8cacd6c03b29 2024-11-20T02:55:08,282 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T02:55:08,321 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T02:55:08,321 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T02:55:08,333 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T02:55:08,333 DEBUG [RS:2;1c82b83631a2:46017 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66367620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1c82b83631a2/172.17.0.2:0 2024-11-20T02:55:08,337 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,337 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a13e31ee199df694a93917b8db953fb, NAME => 'testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.', STARTKEY => '', ENDKEY => ''} 2024-11-20T02:55:08,337 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732071306029 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,338 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T02:55:08,338 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,338 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,339 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,339 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a13e31ee199df694a93917b8db953fb columnFamilyName family 2024-11-20T02:55:08,340 DEBUG [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T02:55:08,341 DEBUG [StoreFileOpener-1a13e31ee199df694a93917b8db953fb-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c, delay=992 ms 2024-11-20T02:55:08,343 DEBUG [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:08,343 INFO [StoreOpener-1a13e31ee199df694a93917b8db953fb-1 {}] regionserver.HStore(327): Store=1a13e31ee199df694a93917b8db953fb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T02:55:08,343 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,344 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,344 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,344 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,344 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,345 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,346 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1a13e31ee199df694a93917b8db953fb; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68974096, jitterRate=0.02779412269592285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T02:55:08,346 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,346 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;1c82b83631a2:46017 2024-11-20T02:55:08,346 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T02:55:08,346 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T02:55:08,346 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T02:55:08,347 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1a13e31ee199df694a93917b8db953fb: Running coprocessor pre-open hook at 1732071308338Writing region info on filesystem at 1732071308338Initializing all the Stores at 1732071308338Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732071308338Cleaning up temporary data from old regions at 1732071308344 (+6 ms)Running coprocessor post-open hooks at 1732071308346 (+2 ms)Region opened successfully at 1732071308347 (+1 ms) 2024-11-20T02:55:08,347 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(2659): reportForDuty to master=1c82b83631a2,41191,1732071304462 with port=46017, startcode=1732071308231 2024-11-20T02:55:08,347 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb., pid=9, masterSystemTime=1732071308334 2024-11-20T02:55:08,348 DEBUG [RS:2;1c82b83631a2:46017 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T02:55:08,350 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T02:55:08,350 DEBUG [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,350 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,350 INFO [RS_OPEN_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,351 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41191 {}] master.ServerManager(517): Registering regionserver=1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,351 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1a13e31ee199df694a93917b8db953fb, regionState=OPEN, openSeqNum=10, regionLocation=1c82b83631a2,38797,1732071304625 2024-11-20T02:55:08,352 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98 2024-11-20T02:55:08,352 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T02:55:08,352 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T02:55:08,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,38797,1732071304625 because future has completed 2024-11-20T02:55:08,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T02:55:08,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a13e31ee199df694a93917b8db953fb, server=1c82b83631a2,38797,1732071304625 in 173 msec 2024-11-20T02:55:08,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T02:55:08,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732071306029, region=1a13e31ee199df694a93917b8db953fb, ASSIGN in 336 msec 2024-11-20T02:55:08,359 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 1c82b83631a2,34645,1732071304667 after splitting done 2024-11-20T02:55:08,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 1c82b83631a2,34645,1732071304667, splitWal=true, meta=false in 352 msec 2024-11-20T02:55:08,362 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:08,362 DEBUG [RS:2;1c82b83631a2:46017 {}] zookeeper.ZKUtil(111): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,362 WARN [RS:2;1c82b83631a2:46017 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T02:55:08,362 INFO [RS:2;1c82b83631a2:46017 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T02:55:08,363 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1c82b83631a2,46017,1732071308231] 2024-11-20T02:55:08,363 DEBUG [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,368 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T02:55:08,370 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T02:55:08,370 INFO [RS:2;1c82b83631a2:46017 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T02:55:08,370 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,371 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T02:55:08,371 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T02:55:08,371 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,371 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,371 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,371 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,371 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1c82b83631a2:0, corePoolSize=2, maxPoolSize=2 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1c82b83631a2:0, corePoolSize=1, maxPoolSize=1 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:08,372 DEBUG [RS:2;1c82b83631a2:46017 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1c82b83631a2:0, corePoolSize=3, maxPoolSize=3 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,373 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,46017,1732071308231-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T02:55:08,389 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T02:55:08,390 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=1c82b83631a2,46017,1732071308231-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,390 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,390 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.Replication(171): 1c82b83631a2,46017,1732071308231 started 2024-11-20T02:55:08,408 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T02:55:08,408 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1482): Serving as 1c82b83631a2,46017,1732071308231, RpcServer on 1c82b83631a2/172.17.0.2:46017, sessionid=0x10157743fc00003 2024-11-20T02:55:08,408 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T02:55:08,408 DEBUG [RS:2;1c82b83631a2:46017 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,408 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,46017,1732071308231' 2024-11-20T02:55:08,408 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T02:55:08,409 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1c82b83631a2,46017,1732071308231' 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T02:55:08,410 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T02:55:08,411 DEBUG [RS:2;1c82b83631a2:46017 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T02:55:08,411 INFO [RS:2;1c82b83631a2:46017 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T02:55:08,411 INFO [RS:2;1c82b83631a2:46017 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T02:55:08,512 INFO [RS:2;1c82b83631a2:46017 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T02:55:08,513 INFO [RS:2;1c82b83631a2:46017 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=1c82b83631a2%2C46017%2C1732071308231, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,46017,1732071308231, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs, maxLogs=55 2024-11-20T02:55:08,532 INFO [RS:2;1c82b83631a2:46017 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/WALs/1c82b83631a2,46017,1732071308231/1c82b83631a2%2C46017%2C1732071308231.1732071308513 2024-11-20T02:55:08,532 DEBUG [RS:2;1c82b83631a2:46017 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T02:55:08,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T02:55:08,908 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:55:08,908 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:08,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T02:55:08,909 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T02:55:08,909 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=628709626, stopped=false 2024-11-20T02:55:08,909 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1c82b83631a2,41191,1732071304462 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:08,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:08,965 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:55:08,965 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T02:55:08,965 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:08,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:08,966 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,38797,1732071304625' ***** 2024-11-20T02:55:08,966 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T02:55:08,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:08,966 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1c82b83631a2,46017,1732071308231' ***** 2024-11-20T02:55:08,966 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T02:55:08,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T02:55:08,966 INFO [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T02:55:08,966 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T02:55:08,966 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T02:55:08,966 INFO [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T02:55:08,967 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T02:55:08,967 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T02:55:08,967 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T02:55:08,967 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T02:55:08,967 INFO [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence 2024-11-20T02:55:08,967 INFO [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T02:55:08,967 INFO [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/50368d90-4e94-d9e5-f338-a6add899f5a9/bucket.persistence 2024-11-20T02:55:08,967 INFO [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T02:55:08,967 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T02:55:08,967 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T02:55:08,973 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:55:08,974 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:08,977 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1487): Thread RS:2;1c82b83631a2:46017 started persisting bucket cache to file 2024-11-20T02:55:08,977 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1487): Thread RS:0;1c82b83631a2:38797 started persisting bucket cache to file 2024-11-20T02:55:08,977 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:08,977 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T02:55:08,977 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:08,977 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T02:55:08,981 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T02:55:08,981 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T02:55:08,981 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732071308977 2024-11-20T02:55:08,981 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732071308977 2024-11-20T02:55:08,981 DEBUG [RS:2;1c82b83631a2:46017 {}] bucket.BucketCache(1509): Thread RS:2;1c82b83631a2:46017 finished persisting bucket cache to file, renaming 2024-11-20T02:55:08,981 DEBUG [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1509): Thread RS:0;1c82b83631a2:38797 finished persisting bucket cache to file, renaming 2024-11-20T02:55:08,981 WARN [RS:0;1c82b83631a2:38797 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:55:08,981 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:55:08,981 INFO [RS:0;1c82b83631a2:38797 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T02:55:08,981 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:55:08,981 INFO [RS:0;1c82b83631a2:38797 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,46017,1732071308231 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:08,981 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(3091): Received CLOSE for 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,981 INFO [RS:2;1c82b83631a2:46017 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;1c82b83631a2:46017. 2024-11-20T02:55:08,981 DEBUG [RS:2;1c82b83631a2:46017 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:08,981 DEBUG [RS:2;1c82b83631a2:46017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(959): stopping server 1c82b83631a2,38797,1732071304625 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:08,982 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,46017,1732071308231; all regions closed. 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1c82b83631a2:38797. 2024-11-20T02:55:08,982 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T02:55:08,982 DEBUG [RS:0;1c82b83631a2:38797 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T02:55:08,982 DEBUG [RS:0;1c82b83631a2:38797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,982 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1a13e31ee199df694a93917b8db953fb, disabling compactions & flushes 2024-11-20T02:55:08,982 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:55:08,982 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:55:08,982 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. after waiting 0 ms 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:55:08,982 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,982 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T02:55:08,983 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T02:55:08,984 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1a13e31ee199df694a93917b8db953fb=testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb.} 2024-11-20T02:55:08,984 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T02:55:08,984 DEBUG [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1a13e31ee199df694a93917b8db953fb 2024-11-20T02:55:08,984 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T02:55:08,984 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T02:55:08,984 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T02:55:08,984 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T02:55:08,984 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T02:55:08,984 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-20T02:55:08,985 DEBUG [StoreFileCloser-1a13e31ee199df694a93917b8db953fb-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/family/79be5daf5f31443080fca040f08b682c 2024-11-20T02:55:08,986 DEBUG [RS:2;1c82b83631a2:46017 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:08,986 INFO [RS:2;1c82b83631a2:46017 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C46017%2C1732071308231:(num 1732071308513) 2024-11-20T02:55:08,986 DEBUG [RS:2;1c82b83631a2:46017 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T02:55:08,987 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:08,987 INFO [RS:2;1c82b83631a2:46017 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46017 2024-11-20T02:55:08,991 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/default/testBlockEvictionOnGracefulStop1732071306029/1a13e31ee199df694a93917b8db953fb/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T02:55:08,991 INFO [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,991 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1a13e31ee199df694a93917b8db953fb: Waiting for close lock at 1732071308982Running coprocessor pre-close hooks at 1732071308982Disabling compacts and flushes for region at 1732071308982Disabling writes for close at 1732071308982Writing region close event to WAL at 1732071308986 (+4 ms)Running coprocessor post-close hooks at 1732071308991 (+5 ms)Closed at 1732071308991 2024-11-20T02:55:08,992 DEBUG [RS_CLOSE_REGION-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb. 2024-11-20T02:55:08,996 INFO [regionserver/1c82b83631a2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:08,997 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2fd728771cb6452b845d73c507fb72a3 2024-11-20T02:55:09,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,46017,1732071308231 2024-11-20T02:55:09,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:09,011 INFO [RS:2;1c82b83631a2:46017 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:09,013 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/2fd728771cb6452b845d73c507fb72a3 is 209, key is testBlockEvictionOnGracefulStop1732071306029,,1732071306037.1a13e31ee199df694a93917b8db953fb./info:regioninfo/1732071308351/Put/seqid=0 2024-11-20T02:55:09,014 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/2fd728771cb6452b845d73c507fb72a3 2024-11-20T02:55:09,023 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 03c13516ad324c2398e3b6c4527a7a0c 2024-11-20T02:55:09,023 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,46017,1732071308231] 2024-11-20T02:55:09,031 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,46017,1732071308231 already deleted, retry=false 2024-11-20T02:55:09,031 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,46017,1732071308231 expired; onlineServers=1 2024-11-20T02:55:09,039 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/03c13516ad324c2398e3b6c4527a7a0c is 43, key is default/ns:d/1732071305926/Put/seqid=0 2024-11-20T02:55:09,040 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/03c13516ad324c2398e3b6c4527a7a0c 2024-11-20T02:55:09,049 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0bd6512b5c764417a9b129d7126666db 2024-11-20T02:55:09,064 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/0bd6512b5c764417a9b129d7126666db is 80, key is testBlockEvictionOnGracefulStop1732071306029/table:state/1732071306431/Put/seqid=0 2024-11-20T02:55:09,065 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/0bd6512b5c764417a9b129d7126666db 2024-11-20T02:55:09,067 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/info/2fd728771cb6452b845d73c507fb72a3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/2fd728771cb6452b845d73c507fb72a3 2024-11-20T02:55:09,069 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/2fd728771cb6452b845d73c507fb72a3, delay=971 ms 2024-11-20T02:55:09,071 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/2fd728771cb6452b845d73c507fb72a3, entries=14, sequenceid=13, filesize=7.5 K 2024-11-20T02:55:09,071 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/ns/03c13516ad324c2398e3b6c4527a7a0c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/03c13516ad324c2398e3b6c4527a7a0c 2024-11-20T02:55:09,073 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/03c13516ad324c2398e3b6c4527a7a0c, delay=963 ms 2024-11-20T02:55:09,074 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/03c13516ad324c2398e3b6c4527a7a0c, entries=2, sequenceid=13, filesize=5.0 K 2024-11-20T02:55:09,075 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/.tmp/table/0bd6512b5c764417a9b129d7126666db as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/0bd6512b5c764417a9b129d7126666db 2024-11-20T02:55:09,076 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/0bd6512b5c764417a9b129d7126666db, delay=978 ms 2024-11-20T02:55:09,077 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/0bd6512b5c764417a9b129d7126666db, entries=2, sequenceid=13, filesize=5.3 K 2024-11-20T02:55:09,078 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=13, compaction requested=false 2024-11-20T02:55:09,080 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/info/2fd728771cb6452b845d73c507fb72a3 2024-11-20T02:55:09,080 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/ns/03c13516ad324c2398e3b6c4527a7a0c 2024-11-20T02:55:09,081 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/table/0bd6512b5c764417a9b129d7126666db 2024-11-20T02:55:09,086 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-20T02:55:09,087 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T02:55:09,087 INFO [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:09,087 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732071308984Running coprocessor pre-close hooks at 1732071308984Disabling compacts and flushes for region at 1732071308984Disabling writes for close at 1732071308984Obtaining lock to block concurrent updates at 1732071308984Preparing flush snapshotting stores in 1588230740 at 1732071308984Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732071308985 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732071308985Flushing 1588230740/info: creating writer at 1732071308985Flushing 1588230740/info: appending metadata at 1732071309013 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732071309013Flushing 1588230740/ns: creating writer at 1732071309017 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732071309038 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732071309038Flushing 1588230740/table: creating writer at 1732071309042 (+4 ms)Flushing 1588230740/table: appending metadata at 1732071309063 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732071309063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@137d75a3: reopening flushed file at 1732071309067 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24b0478c: reopening flushed file at 1732071309071 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c8e86c8: reopening flushed file at 1732071309074 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 94ms, sequenceid=13, compaction requested=false at 1732071309078 (+4 ms)Writing region close event to WAL at 1732071309082 (+4 ms)Running coprocessor post-close hooks at 1732071309086 (+4 ms)Closed at 1732071309087 (+1 ms) 2024-11-20T02:55:09,087 DEBUG [RS_CLOSE_META-regionserver/1c82b83631a2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T02:55:09,123 INFO [RS:2;1c82b83631a2:46017 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:09,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,124 INFO [RS:2;1c82b83631a2:46017 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,46017,1732071308231; zookeeper connection closed. 2024-11-20T02:55:09,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46017-0x10157743fc00003, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,124 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77e9994c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77e9994c 2024-11-20T02:55:09,184 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(976): stopping server 1c82b83631a2,38797,1732071304625; all regions closed. 2024-11-20T02:55:09,187 DEBUG [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:09,187 INFO [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C38797%2C1732071304625.meta:.meta(num 1732071305847) 2024-11-20T02:55:09,189 DEBUG [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/oldWALs 2024-11-20T02:55:09,190 INFO [RS:0;1c82b83631a2:38797 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1c82b83631a2%2C38797%2C1732071304625:(num 1732071305348) 2024-11-20T02:55:09,190 DEBUG [RS:0;1c82b83631a2:38797 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T02:55:09,190 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T02:55:09,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:09,190 INFO [RS:0;1c82b83631a2:38797 {}] hbase.ChoreService(370): Chore service for: regionserver/1c82b83631a2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:09,191 INFO [RS:0;1c82b83631a2:38797 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:09,191 INFO [regionserver/1c82b83631a2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:09,191 INFO [RS:0;1c82b83631a2:38797 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38797 2024-11-20T02:55:09,223 INFO [RS:0;1c82b83631a2:38797 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:09,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T02:55:09,223 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1c82b83631a2,38797,1732071304625 2024-11-20T02:55:09,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1c82b83631a2,38797,1732071304625] 2024-11-20T02:55:09,240 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1c82b83631a2,38797,1732071304625 already deleted, retry=false 2024-11-20T02:55:09,240 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1c82b83631a2,38797,1732071304625 expired; onlineServers=0 2024-11-20T02:55:09,240 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1c82b83631a2,41191,1732071304462' ***** 2024-11-20T02:55:09,240 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T02:55:09,240 INFO [M:0;1c82b83631a2:41191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T02:55:09,240 INFO [M:0;1c82b83631a2:41191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T02:55:09,240 DEBUG [M:0;1c82b83631a2:41191 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T02:55:09,240 DEBUG [M:0;1c82b83631a2:41191 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T02:55:09,240 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T02:55:09,240 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071305133 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.large.0-1732071305133,5,FailOnTimeoutGroup] 2024-11-20T02:55:09,240 DEBUG [master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071305136 {}] cleaner.HFileCleaner(306): Exit Thread[master/1c82b83631a2:0:becomeActiveMaster-HFileCleaner.small.0-1732071305136,5,FailOnTimeoutGroup] 2024-11-20T02:55:09,241 INFO [M:0;1c82b83631a2:41191 {}] hbase.ChoreService(370): Chore service for: master/1c82b83631a2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T02:55:09,241 INFO [M:0;1c82b83631a2:41191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T02:55:09,241 DEBUG [M:0;1c82b83631a2:41191 {}] master.HMaster(1795): Stopping service threads 2024-11-20T02:55:09,241 INFO [M:0;1c82b83631a2:41191 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T02:55:09,241 INFO [M:0;1c82b83631a2:41191 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T02:55:09,241 INFO [M:0;1c82b83631a2:41191 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T02:55:09,241 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T02:55:09,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T02:55:09,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T02:55:09,248 DEBUG [M:0;1c82b83631a2:41191 {}] zookeeper.ZKUtil(347): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T02:55:09,248 WARN [M:0;1c82b83631a2:41191 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T02:55:09,249 INFO [M:0;1c82b83631a2:41191 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/.lastflushedseqids 2024-11-20T02:55:09,256 INFO [M:0;1c82b83631a2:41191 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T02:55:09,256 INFO [M:0;1c82b83631a2:41191 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T02:55:09,256 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T02:55:09,256 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:09,256 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:09,256 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T02:55:09,256 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:09,256 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-20T02:55:09,272 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4c3e3c9f1d984450bd691f5a35964c2c 2024-11-20T02:55:09,290 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c3e3c9f1d984450bd691f5a35964c2c is 82, key is hbase:meta,,1/info:regioninfo/1732071305886/Put/seqid=0 2024-11-20T02:55:09,291 INFO [M:0;1c82b83631a2:41191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c3e3c9f1d984450bd691f5a35964c2c 2024-11-20T02:55:09,301 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 924705e1285b49eea9bc45435a30e179 2024-11-20T02:55:09,321 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/924705e1285b49eea9bc45435a30e179 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732071306435/Put/seqid=0 2024-11-20T02:55:09,322 INFO [M:0;1c82b83631a2:41191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/924705e1285b49eea9bc45435a30e179 2024-11-20T02:55:09,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,332 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3dc40ce1debb45efb3a3717362f522b4 2024-11-20T02:55:09,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38797-0x10157743fc00001, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,332 INFO [RS:0;1c82b83631a2:38797 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:09,332 INFO [RS:0;1c82b83631a2:38797 {}] regionserver.HRegionServer(1031): Exiting; stopping=1c82b83631a2,38797,1732071304625; zookeeper connection closed. 2024-11-20T02:55:09,332 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@21ee316c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@21ee316c 2024-11-20T02:55:09,332 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T02:55:09,349 DEBUG [M:0;1c82b83631a2:41191 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dc40ce1debb45efb3a3717362f522b4 is 69, key is 1c82b83631a2,38797,1732071304625/rs:state/1732071305161/Put/seqid=0 2024-11-20T02:55:09,349 INFO [M:0;1c82b83631a2:41191 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dc40ce1debb45efb3a3717362f522b4 2024-11-20T02:55:09,351 INFO [M:0;1c82b83631a2:41191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dc40ce1debb45efb3a3717362f522b4 2024-11-20T02:55:09,352 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4c3e3c9f1d984450bd691f5a35964c2c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4c3e3c9f1d984450bd691f5a35964c2c 2024-11-20T02:55:09,355 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4c3e3c9f1d984450bd691f5a35964c2c, entries=8, sequenceid=83, filesize=5.5 K 2024-11-20T02:55:09,356 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/924705e1285b49eea9bc45435a30e179 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/924705e1285b49eea9bc45435a30e179 2024-11-20T02:55:09,358 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/924705e1285b49eea9bc45435a30e179, entries=9, sequenceid=83, filesize=6.4 K 2024-11-20T02:55:09,359 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3dc40ce1debb45efb3a3717362f522b4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3dc40ce1debb45efb3a3717362f522b4 2024-11-20T02:55:09,361 INFO [M:0;1c82b83631a2:41191 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3dc40ce1debb45efb3a3717362f522b4 2024-11-20T02:55:09,362 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/42229fbd-b602-af69-4e66-f3e6bc48975e/d2e8de7f-6928-ed41-22f9-01b589b5ed98/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3dc40ce1debb45efb3a3717362f522b4, entries=3, sequenceid=83, filesize=5.3 K 2024-11-20T02:55:09,363 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=83, compaction requested=false 2024-11-20T02:55:09,375 INFO [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T02:55:09,375 DEBUG [M:0;1c82b83631a2:41191 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732071309256Disabling compacts and flushes for region at 1732071309256Disabling writes for close at 1732071309256Obtaining lock to block concurrent updates at 1732071309256Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732071309256Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1732071309257 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732071309257Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732071309257Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732071309289 (+32 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732071309290 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732071309293 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732071309320 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732071309321 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732071309324 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732071309348 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732071309348Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@488dac0: reopening flushed file at 1732071309352 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@692559f0: reopening flushed file at 1732071309355 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1db1c9e4: reopening flushed file at 1732071309359 (+4 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 106ms, sequenceid=83, compaction requested=false at 1732071309363 (+4 ms)Writing region close event to WAL at 1732071309375 (+12 ms)Closed at 1732071309375 2024-11-20T02:55:09,376 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T02:55:09,376 INFO [M:0;1c82b83631a2:41191 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T02:55:09,376 INFO [M:0;1c82b83631a2:41191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41191 2024-11-20T02:55:09,377 INFO [M:0;1c82b83631a2:41191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T02:55:09,504 INFO [M:0;1c82b83631a2:41191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T02:55:09,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41191-0x10157743fc00000, quorum=127.0.0.1:50458, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T02:55:09,507 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T02:55:09,508 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T02:55:09,508 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T02:55:09,515 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 50) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 375), ProcessCount=11 (was 11), AvailableMemoryMB=6752 (was 6916)