2024-11-18 18:55:14,454 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 18:55:14,473 main DEBUG Took 0.017344 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 18:55:14,474 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 18:55:14,474 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 18:55:14,475 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 18:55:14,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,485 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 18:55:14,502 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,504 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,505 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,506 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,507 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,507 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,509 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,510 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,511 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,511 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,513 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,514 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,514 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,515 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,516 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,516 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,517 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,518 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,518 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,519 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,519 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,520 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,521 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,521 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 18:55:14,522 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,522 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 18:55:14,524 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 18:55:14,526 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 18:55:14,529 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 18:55:14,530 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 18:55:14,531 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 18:55:14,532 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 18:55:14,544 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 18:55:14,548 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 18:55:14,550 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 18:55:14,551 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 18:55:14,551 main DEBUG createAppenders(={Console}) 2024-11-18 18:55:14,553 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-18 18:55:14,553 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 18:55:14,553 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-18 18:55:14,554 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 18:55:14,554 main DEBUG OutputStream closed 2024-11-18 18:55:14,555 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 18:55:14,555 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 18:55:14,556 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-18 18:55:14,652 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 18:55:14,655 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 18:55:14,657 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 18:55:14,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 18:55:14,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 18:55:14,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 18:55:14,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 18:55:14,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 18:55:14,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 18:55:14,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 18:55:14,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 18:55:14,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 18:55:14,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 18:55:14,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 18:55:14,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 18:55:14,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 18:55:14,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 18:55:14,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 18:55:14,667 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 18:55:14,668 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-18 18:55:14,668 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 18:55:14,669 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-18T18:55:14,681 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-18 18:55:14,684 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 18:55:14,684 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T18:55:14,906 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea 2024-11-18T18:55:14,932 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=77, ProcessCount=11, AvailableMemoryMB=5514 2024-11-18T18:55:15,027 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T18:55:15,120 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:15,135 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/cluster_377e754b-1061-bae7-69c9-e849d124fb7f, deleteOnExit=true 2024-11-18T18:55:15,245 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/cluster_377e754b-1061-bae7-69c9-e849d124fb7f/zookeeper_0, clientPort=56408, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/cluster_377e754b-1061-bae7-69c9-e849d124fb7f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/cluster_377e754b-1061-bae7-69c9-e849d124fb7f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T18:55:15,257 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56408 2024-11-18T18:55:15,258 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:15,343 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 with version=8 2024-11-18T18:55:15,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase-staging 2024-11-18T18:55:15,435 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T18:55:15,676 INFO [Time-limited test {}] client.ConnectionUtils(128): master/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:15,685 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:15,687 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:15,694 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:15,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:15,695 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:15,826 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T18:55:15,912 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:55:15,957 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T18:55:15,969 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T18:55:15,973 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:16,005 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 76897 (auto-detected) 2024-11-18T18:55:16,006 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T18:55:16,024 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33743 2024-11-18T18:55:16,045 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33743 connecting to ZooKeeper ensemble=127.0.0.1:56408 2024-11-18T18:55:16,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:337430x0, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:16,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33743-0x1015096b9020000 connected 2024-11-18T18:55:16,209 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,211 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:16,227 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26, hbase.cluster.distributed=false 2024-11-18T18:55:16,252 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:16,259 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33743 2024-11-18T18:55:16,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33743 2024-11-18T18:55:16,260 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33743 2024-11-18T18:55:16,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33743 2024-11-18T18:55:16,261 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33743 2024-11-18T18:55:16,341 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:16,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,342 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,343 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:16,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,343 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:16,346 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:55:16,350 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:16,351 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35107 2024-11-18T18:55:16,353 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35107 connecting to ZooKeeper ensemble=127.0.0.1:56408 2024-11-18T18:55:16,353 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,354 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351070x0, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:16,365 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35107-0x1015096b9020001 connected 2024-11-18T18:55:16,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:16,371 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:55:16,384 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.cache 2024-11-18T18:55:16,391 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T18:55:16,393 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T18:55:16,395 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T18:55:16,397 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T18:55:16,397 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T18:55:16,404 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T18:55:16,405 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T18:55:16,405 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence is complete. 2024-11-18T18:55:16,414 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:55:16,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:55:16,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:16,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35107 2024-11-18T18:55:16,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35107 2024-11-18T18:55:16,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35107 2024-11-18T18:55:16,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35107 2024-11-18T18:55:16,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35107 2024-11-18T18:55:16,454 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:16,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,454 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,455 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:16,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:16,455 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:16,456 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:55:16,456 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:16,458 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39895 2024-11-18T18:55:16,461 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39895 connecting to ZooKeeper ensemble=127.0.0.1:56408 2024-11-18T18:55:16,462 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,463 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:398950x0, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:16,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:398950x0, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:16,474 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39895-0x1015096b9020002 connected 2024-11-18T18:55:16,475 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:55:16,476 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.cache 2024-11-18T18:55:16,477 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T18:55:16,477 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T18:55:16,478 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T18:55:16,478 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T18:55:16,479 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T18:55:16,479 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T18:55:16,479 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:55:16,480 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T18:55:16,480 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence is complete. 2024-11-18T18:55:16,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:55:16,484 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:16,488 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39895 2024-11-18T18:55:16,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39895 2024-11-18T18:55:16,492 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39895 2024-11-18T18:55:16,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39895 2024-11-18T18:55:16,496 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39895 2024-11-18T18:55:16,514 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;39fff3b0f89c:33743 2024-11-18T18:55:16,515 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:16,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,534 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:16,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:16,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:16,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,574 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T18:55:16,576 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/39fff3b0f89c,33743,1731956115513 from backup master directory 2024-11-18T18:55:16,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:16,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:16,589 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:16,589 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:16,591 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T18:55:16,593 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T18:55:16,600 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase.id] with ID: 63f7555f-031a-453c-9688-6ef60c6691c8 2024-11-18T18:55:16,601 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.tmp/hbase.id 2024-11-18T18:55:16,606 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase.id] 2024-11-18T18:55:16,642 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:16,647 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T18:55:16,668 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-18T18:55:16,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:16,715 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:55:16,719 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T18:55:16,768 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:55:16,775 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:16,826 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store 2024-11-18T18:55:16,846 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T18:55:16,849 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:16,850 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:55:16,850 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:16,851 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:16,852 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:55:16,852 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:16,852 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:16,853 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956116850Disabling compacts and flushes for region at 1731956116850Disabling writes for close at 1731956116852 (+2 ms)Writing region close event to WAL at 1731956116852Closed at 1731956116852 2024-11-18T18:55:16,854 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/.initializing 2024-11-18T18:55:16,854 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:16,865 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:16,878 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C33743%2C1731956115513, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,33743,1731956115513, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/oldWALs, maxLogs=10 2024-11-18T18:55:16,910 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,33743,1731956115513/39fff3b0f89c%2C33743%2C1731956115513.1731956116882 2024-11-18T18:55:16,910 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:16,911 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:16,911 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:16,915 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:16,916 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:16,949 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:16,978 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T18:55:16,984 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:16,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:16,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:16,991 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T18:55:16,992 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:16,992 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:16,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:16,998 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T18:55:16,998 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:16,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:16,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,004 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T18:55:17,004 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:17,005 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,009 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,009 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,012 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,013 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,015 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:17,018 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:17,025 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:17,026 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60097512, jitterRate=-0.10447728633880615}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:17,037 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731956116927Initializing all the Stores at 1731956116928 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956116928Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956116929 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956116929Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956116930 (+1 ms)Cleaning up temporary data from old regions at 1731956117013 (+83 ms)Region opened successfully at 1731956117037 (+24 ms) 2024-11-18T18:55:17,039 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T18:55:17,077 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:17,109 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T18:55:17,121 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T18:55:17,121 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T18:55:17,124 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:55:17,124 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:55:17,130 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T18:55:17,130 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T18:55:17,156 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T18:55:17,166 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T18:55:17,197 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T18:55:17,200 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T18:55:17,202 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T18:55:17,213 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T18:55:17,217 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T18:55:17,222 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T18:55:17,231 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T18:55:17,233 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T18:55:17,239 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T18:55:17,266 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T18:55:17,272 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T18:55:17,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:17,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:17,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:17,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,294 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=39fff3b0f89c,33743,1731956115513, sessionid=0x1015096b9020000, setting cluster-up flag (Was=false) 2024-11-18T18:55:17,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,322 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,347 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T18:55:17,350 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:17,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:17,389 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T18:55:17,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:17,397 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T18:55:17,402 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(746): ClusterId : 63f7555f-031a-453c-9688-6ef60c6691c8 2024-11-18T18:55:17,402 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(746): ClusterId : 63f7555f-031a-453c-9688-6ef60c6691c8 2024-11-18T18:55:17,404 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:55:17,404 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:55:17,416 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:55:17,416 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:55:17,416 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:55:17,416 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:55:17,424 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:55:17,424 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:55:17,424 DEBUG [RS:0;39fff3b0f89c:35107 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5368b849, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:17,424 DEBUG [RS:1;39fff3b0f89c:39895 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab072ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:17,439 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;39fff3b0f89c:35107 2024-11-18T18:55:17,442 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:55:17,443 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:55:17,443 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:55:17,444 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;39fff3b0f89c:39895 2024-11-18T18:55:17,445 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:55:17,445 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:55:17,445 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:55:17,445 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,33743,1731956115513 with port=35107, startcode=1731956116307 2024-11-18T18:55:17,447 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,33743,1731956115513 with port=39895, startcode=1731956116453 2024-11-18T18:55:17,461 DEBUG [RS:1;39fff3b0f89c:39895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:55:17,461 DEBUG [RS:0;39fff3b0f89c:35107 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:55:17,487 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:17,504 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T18:55:17,514 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T18:55:17,519 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 39fff3b0f89c,33743,1731956115513 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T18:55:17,529 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:17,529 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:17,529 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:17,529 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:17,530 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/39fff3b0f89c:0, corePoolSize=10, maxPoolSize=10 2024-11-18T18:55:17,530 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,530 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:55:17,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34873, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:55:17,530 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:17,530 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,534 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731956147534 2024-11-18T18:55:17,535 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:17,535 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T18:55:17,536 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T18:55:17,537 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T18:55:17,537 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:55:17,539 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,539 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T18:55:17,541 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T18:55:17,541 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T18:55:17,542 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T18:55:17,542 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T18:55:17,544 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-18T18:55:17,543 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,552 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T18:55:17,553 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T18:55:17,554 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T18:55:17,559 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T18:55:17,560 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T18:55:17,562 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956117561,5,FailOnTimeoutGroup] 2024-11-18T18:55:17,563 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956117563,5,FailOnTimeoutGroup] 2024-11-18T18:55:17,563 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,563 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T18:55:17,565 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,565 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,566 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T18:55:17,566 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-18T18:55:17,566 WARN [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T18:55:17,566 WARN [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-18T18:55:17,572 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T18:55:17,573 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:17,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:17,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T18:55:17,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:55:17,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:17,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T18:55:17,593 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:55:17,593 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:17,594 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T18:55:17,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:55:17,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:17,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T18:55:17,601 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:55:17,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:17,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:17,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:55:17,603 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:17,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:17,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:55:17,604 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:55:17,604 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:17,605 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:55:17,612 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:17,612 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59064440, jitterRate=-0.11987125873565674}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:17,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731956117583Initializing all the Stores at 1731956117584 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956117584Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956117584Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956117584Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956117584Cleaning up temporary data from old regions at 1731956117604 (+20 ms)Region opened successfully at 1731956117616 (+12 ms) 2024-11-18T18:55:17,616 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:55:17,616 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:55:17,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:55:17,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:55:17,617 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:55:17,618 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:17,618 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956117616Disabling compacts and flushes for region at 1731956117616Disabling writes for close at 1731956117617 (+1 ms)Writing region close event to WAL at 1731956117618 (+1 ms)Closed at 1731956117618 2024-11-18T18:55:17,621 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:17,621 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T18:55:17,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:55:17,634 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:17,636 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T18:55:17,668 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,33743,1731956115513 with port=39895, startcode=1731956116453 2024-11-18T18:55:17,668 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,33743,1731956115513 with port=35107, startcode=1731956116307 2024-11-18T18:55:17,671 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,673 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,680 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,680 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,680 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:17,680 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T18:55:17,680 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:55:17,682 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:17,682 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T18:55:17,683 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:55:17,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:17,722 DEBUG [RS:1;39fff3b0f89c:39895 {}] zookeeper.ZKUtil(111): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,722 WARN [RS:1;39fff3b0f89c:39895 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:17,723 INFO [RS:1;39fff3b0f89c:39895 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:17,723 DEBUG [RS:0;39fff3b0f89c:35107 {}] zookeeper.ZKUtil(111): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,723 WARN [RS:0;39fff3b0f89c:35107 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:17,723 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,723 INFO [RS:0;39fff3b0f89c:35107 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:17,723 DEBUG [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,724 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,39895,1731956116453] 2024-11-18T18:55:17,724 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,35107,1731956116307] 2024-11-18T18:55:17,746 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:55:17,746 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:55:17,763 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:55:17,763 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:55:17,768 INFO [RS:0;39fff3b0f89c:35107 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:55:17,768 INFO [RS:1;39fff3b0f89c:39895 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:55:17,768 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,768 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,769 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:55:17,770 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:55:17,774 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:55:17,774 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:55:17,775 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,775 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,775 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,775 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,775 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,775 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,775 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,776 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:17,777 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:0;39fff3b0f89c:35107 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:17,777 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:17,777 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:17,778 DEBUG [RS:1;39fff3b0f89c:39895 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:17,778 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,35107,1731956116307-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,779 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39895,1731956116453-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:17,787 WARN [39fff3b0f89c:33743 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:55:17,803 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:55:17,803 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:55:17,805 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,35107,1731956116307-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,805 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,39895,1731956116453-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,805 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,806 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,806 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.Replication(171): 39fff3b0f89c,35107,1731956116307 started 2024-11-18T18:55:17,806 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.Replication(171): 39fff3b0f89c,39895,1731956116453 started 2024-11-18T18:55:17,832 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,832 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:17,832 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,39895,1731956116453, RpcServer on 39fff3b0f89c/172.17.0.2:39895, sessionid=0x1015096b9020002 2024-11-18T18:55:17,832 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,35107,1731956116307, RpcServer on 39fff3b0f89c/172.17.0.2:35107, sessionid=0x1015096b9020001 2024-11-18T18:55:17,833 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:55:17,833 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:55:17,834 DEBUG [RS:0;39fff3b0f89c:35107 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,834 DEBUG [RS:1;39fff3b0f89c:39895 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,834 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,35107,1731956116307' 2024-11-18T18:55:17,834 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,39895,1731956116453' 2024-11-18T18:55:17,834 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:55:17,834 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:55:17,835 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:55:17,835 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:55:17,836 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:55:17,836 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:55:17,836 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:55:17,836 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:55:17,836 DEBUG [RS:0;39fff3b0f89c:35107 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:17,836 DEBUG [RS:1;39fff3b0f89c:39895 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:17,836 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,35107,1731956116307' 2024-11-18T18:55:17,836 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,39895,1731956116453' 2024-11-18T18:55:17,836 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:55:17,836 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:55:17,837 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:55:17,837 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:55:17,838 DEBUG [RS:0;39fff3b0f89c:35107 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:55:17,838 DEBUG [RS:1;39fff3b0f89c:39895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:55:17,838 INFO [RS:0;39fff3b0f89c:35107 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:55:17,838 INFO [RS:1;39fff3b0f89c:39895 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:55:17,838 INFO [RS:0;39fff3b0f89c:35107 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:55:17,838 INFO [RS:1;39fff3b0f89c:39895 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:55:17,948 INFO [RS:0;39fff3b0f89c:35107 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:17,948 INFO [RS:1;39fff3b0f89c:39895 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:17,950 INFO [RS:0;39fff3b0f89c:35107 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C35107%2C1731956116307, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,35107,1731956116307, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:17,951 INFO [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C39895%2C1731956116453, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,39895,1731956116453, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:17,970 INFO [RS:0;39fff3b0f89c:35107 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,35107,1731956116307/39fff3b0f89c%2C35107%2C1731956116307.1731956117954 2024-11-18T18:55:17,971 DEBUG [RS:0;39fff3b0f89c:35107 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:17,980 INFO [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,39895,1731956116453/39fff3b0f89c%2C39895%2C1731956116453.1731956117955 2024-11-18T18:55:17,980 DEBUG [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:18,039 DEBUG [39fff3b0f89c:33743 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T18:55:18,048 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T18:55:18,051 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:55:18,051 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:55:18,052 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:55:18,052 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:55:18,052 INFO [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:55:18,052 INFO [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:55:18,052 DEBUG [39fff3b0f89c:33743 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:55:18,058 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:18,064 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,39895,1731956116453, state=OPENING 2024-11-18T18:55:18,097 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T18:55:18,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:18,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:18,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:18,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,107 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,109 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:18,111 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,39895,1731956116453}] 2024-11-18T18:55:18,286 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:55:18,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43699, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:55:18,298 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:55:18,299 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:18,299 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:55:18,301 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C39895%2C1731956116453.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,39895,1731956116453, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:18,328 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,39895,1731956116453/39fff3b0f89c%2C39895%2C1731956116453.meta.1731956118304.meta 2024-11-18T18:55:18,328 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:18,329 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:18,330 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:55:18,333 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:55:18,338 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:55:18,343 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:55:18,344 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:18,344 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:55:18,344 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:55:18,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T18:55:18,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:55:18,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:18,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:18,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T18:55:18,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:55:18,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:18,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:18,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T18:55:18,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:55:18,350 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:18,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:18,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T18:55:18,352 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:55:18,352 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:18,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:18,352 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:55:18,353 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:18,354 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:18,354 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:55:18,355 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:55:18,355 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:18,356 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:55:18,357 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68865576, jitterRate=0.026177048683166504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:18,357 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:55:18,358 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731956118345Writing region info on filesystem at 1731956118345Initializing all the Stores at 1731956118346 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956118346Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956118346Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956118346Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956118346Cleaning up temporary data from old regions at 1731956118355 (+9 ms)Running coprocessor post-open hooks at 1731956118357 (+2 ms)Region opened successfully at 1731956118358 (+1 ms) 2024-11-18T18:55:18,364 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731956118278 2024-11-18T18:55:18,376 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:55:18,377 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:55:18,378 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:18,379 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,39895,1731956116453, state=OPEN 2024-11-18T18:55:18,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:18,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:18,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:18,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,389 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:18,390 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:18,393 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T18:55:18,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,39895,1731956116453 in 279 msec 2024-11-18T18:55:18,398 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T18:55:18,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 768 msec 2024-11-18T18:55:18,399 DEBUG [PEWorker-3 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:18,399 INFO [PEWorker-3 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T18:55:18,421 DEBUG [PEWorker-3 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:55:18,423 DEBUG [PEWorker-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,39895,1731956116453, seqNum=-1] 2024-11-18T18:55:18,445 DEBUG [PEWorker-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:55:18,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:55:18,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0290 sec 2024-11-18T18:55:18,467 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731956118467, completionTime=-1 2024-11-18T18:55:18,470 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T18:55:18,470 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:55:18,507 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:55:18,508 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731956178508 2024-11-18T18:55:18,508 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731956238508 2024-11-18T18:55:18,508 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-11-18T18:55:18,511 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T18:55:18,519 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,519 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,519 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,521 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-39fff3b0f89c:33743, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,522 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,525 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,530 DEBUG [master/39fff3b0f89c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T18:55:18,554 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.965sec 2024-11-18T18:55:18,556 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T18:55:18,557 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T18:55:18,558 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T18:55:18,558 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T18:55:18,558 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T18:55:18,559 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:18,559 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T18:55:18,560 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T18:55:18,561 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T18:55:18,561 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,33743,1731956115513-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:18,626 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:18,629 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T18:55:18,629 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T18:55:18,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39fff3b0f89c,33743,-1 for getting cluster id 2024-11-18T18:55:18,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T18:55:18,649 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '63f7555f-031a-453c-9688-6ef60c6691c8' 2024-11-18T18:55:18,652 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T18:55:18,652 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "63f7555f-031a-453c-9688-6ef60c6691c8" 2024-11-18T18:55:18,653 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a75a32d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:18,653 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39fff3b0f89c,33743,-1] 2024-11-18T18:55:18,656 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T18:55:18,658 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:18,659 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37170, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:55:18,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:18,663 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:55:18,672 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,39895,1731956116453, seqNum=-1] 2024-11-18T18:55:18,673 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:55:18,675 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58862, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:55:18,694 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:18,698 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T18:55:18,702 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:18,705 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@194a1284 2024-11-18T18:55:18,706 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T18:55:18,708 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T18:55:18,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731956118694', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:55:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 2024-11-18T18:55:18,721 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:55:18,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731956118694" procId is: 4 2024-11-18T18:55:18,723 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:18,725 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:55:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:18,737 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 94f0062296179ee1a5d0c9fc0ff8d07b, NAME => 'testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731956118694', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:18,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:18,744 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1722): Closing 94f0062296179ee1a5d0c9fc0ff8d07b, disabling compactions & flushes 2024-11-18T18:55:18,744 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:18,744 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:18,744 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. after waiting 0 ms 2024-11-18T18:55:18,744 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:18,744 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:18,744 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731956118694-pool-0 {}] regionserver.HRegion(1676): Region close journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Waiting for close lock at 1731956118743Disabling compacts and flushes for region at 1731956118743Disabling writes for close at 1731956118744 (+1 ms)Writing region close event to WAL at 1731956118744Closed at 1731956118744 2024-11-18T18:55:18,745 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:55:18,750 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731956118746"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956118746"}]},"ts":"1731956118746"} 2024-11-18T18:55:18,754 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:55:18,755 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:55:18,757 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731956118694","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956118755"}]},"ts":"1731956118755"} 2024-11-18T18:55:18,761 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731956118694, state=ENABLING in hbase:meta 2024-11-18T18:55:18,761 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T18:55:18,762 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:55:18,762 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:55:18,762 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:55:18,762 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:55:18,762 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:55:18,762 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:55:18,762 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:55:18,763 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, ASSIGN}] 2024-11-18T18:55:18,765 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, ASSIGN 2024-11-18T18:55:18,766 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, ASSIGN; state=OFFLINE, location=39fff3b0f89c,39895,1731956116453; forceNewPlan=false, retain=false 2024-11-18T18:55:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:18,921 INFO [39fff3b0f89c:33743 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T18:55:18,923 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=OPENING, regionLocation=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:18,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, ASSIGN because future has completed 2024-11-18T18:55:18,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453}] 2024-11-18T18:55:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:19,086 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:19,086 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f0062296179ee1a5d0c9fc0ff8d07b, NAME => 'testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:19,086 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731956118694 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,087 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:19,087 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,087 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,089 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,093 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f0062296179ee1a5d0c9fc0ff8d07b columnFamilyName family 2024-11-18T18:55:19,093 DEBUG [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:19,094 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(327): Store=94f0062296179ee1a5d0c9fc0ff8d07b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:19,094 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,096 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,097 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,097 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,097 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,099 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,105 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:19,106 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 94f0062296179ee1a5d0c9fc0ff8d07b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62413704, jitterRate=-0.06996333599090576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:55:19,106 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:19,108 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Running coprocessor pre-open hook at 1731956119088Writing region info on filesystem at 1731956119088Initializing all the Stores at 1731956119088Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956119088Cleaning up temporary data from old regions at 1731956119097 (+9 ms)Running coprocessor post-open hooks at 1731956119106 (+9 ms)Region opened successfully at 1731956119107 (+1 ms) 2024-11-18T18:55:19,109 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b., pid=6, masterSystemTime=1731956119083 2024-11-18T18:55:19,111 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:19,112 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:19,112 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:19,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453 because future has completed 2024-11-18T18:55:19,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T18:55:19,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453 in 187 msec 2024-11-18T18:55:19,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T18:55:19,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, ASSIGN in 356 msec 2024-11-18T18:55:19,123 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:55:19,123 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731956118694","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956119123"}]},"ts":"1731956119123"} 2024-11-18T18:55:19,126 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731956118694, state=ENABLED in hbase:meta 2024-11-18T18:55:19,127 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:55:19,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731956118694 in 411 msec 2024-11-18T18:55:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:19,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731956118694 get assigned. Timeout = 60000ms 2024-11-18T18:55:19,360 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731956118694 completed 2024-11-18T18:55:19,363 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:55:19,368 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731956118694 assigned to meta. Checking AM states. 2024-11-18T18:55:19,368 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:55:19,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731956118694 assigned. 2024-11-18T18:55:19,377 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731956118694', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b., hostname=39fff3b0f89c,39895,1731956116453, seqNum=2] 2024-11-18T18:55:19,389 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 94f0062296179ee1a5d0c9fc0ff8d07b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T18:55:19,411 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:19,447 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/.tmp/family/4bbc88147a824c3ea826f7a1512e58b5 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T18:55:19,452 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/.tmp/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:19,484 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/.tmp/family/4bbc88147a824c3ea826f7a1512e58b5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:19,485 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, delay=949 ms 2024-11-18T18:55:19,487 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T18:55:19,492 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 94f0062296179ee1a5d0c9fc0ff8d07b in 103ms, sequenceid=6, compaction requested=false 2024-11-18T18:55:19,493 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731956118694' 2024-11-18T18:55:19,495 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 94f0062296179ee1a5d0c9fc0ff8d07b: 2024-11-18T18:55:20,439 DEBUG [hfile-prefetch-1731956119485 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4bbc88147a824c3ea826f7a1512e58b5_0. 2024-11-18T18:55:20,445 DEBUG [hfile-prefetch-1731956119485 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4bbc88147a824c3ea826f7a1512e58b5_117. 2024-11-18T18:55:20,446 DEBUG [hfile-prefetch-1731956119485 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, with total blocks 2, and data blocks 1 2024-11-18T18:55:20,446 DEBUG [hfile-prefetch-1731956119485 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T18:55:20,446 DEBUG [hfile-prefetch-1731956119485 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T18:55:20,447 DEBUG [hfile-prefetch-1731956119485 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5. Will sleep 100ms and try the verification again. 2024-11-18T18:55:20,508 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=94f0062296179ee1a5d0c9fc0ff8d07b, source=39fff3b0f89c,39895,1731956116453, destination=39fff3b0f89c,35107,1731956116307, warming up region on 39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:20,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:55:20,511 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=94f0062296179ee1a5d0c9fc0ff8d07b, source=39fff3b0f89c,39895,1731956116453, destination=39fff3b0f89c,35107,1731956116307, running balancer 2024-11-18T18:55:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE 2024-11-18T18:55:20,512 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE 2024-11-18T18:55:20,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60449, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:55:20,514 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=CLOSING, regionLocation=39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:20,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(7855): Warmup {ENCODED => 94f0062296179ee1a5d0c9fc0ff8d07b, NAME => 'testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:20,516 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:20,517 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f0062296179ee1a5d0c9fc0ff8d07b columnFamilyName family 2024-11-18T18:55:20,517 DEBUG [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:20,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE because future has completed 2024-11-18T18:55:20,518 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-18T18:55:20,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453}] 2024-11-18T18:55:20,520 DEBUG [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:20,521 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(327): Store=94f0062296179ee1a5d0c9fc0ff8d07b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1722): Closing 94f0062296179ee1a5d0c9fc0ff8d07b, disabling compactions & flushes 2024-11-18T18:55:20,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. after waiting 0 ms 2024-11-18T18:55:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,522 DEBUG [StoreFileCloser-94f0062296179ee1a5d0c9fc0ff8d07b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:20,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35107 {}] regionserver.HRegion(1676): Region close journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Waiting for close lock at 1731956120521Disabling compacts and flushes for region at 1731956120521Disabling writes for close at 1731956120521Writing region close event to WAL at 1731956120523 (+2 ms)Closed at 1731956120523 2024-11-18T18:55:20,524 WARN [hfile-prefetch-1731956119485 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T18:55:20,525 DEBUG [hfile-prefetch-1731956119485 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33743 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-18T18:55:20,677 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:20,678 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-18T18:55:20,679 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 94f0062296179ee1a5d0c9fc0ff8d07b, disabling compactions & flushes 2024-11-18T18:55:20,679 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,679 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,679 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. after waiting 0 ms 2024-11-18T18:55:20,679 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,682 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T18:55:20,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T18:55:20,682 DEBUG [StoreFileCloser-94f0062296179ee1a5d0c9fc0ff8d07b-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4bbc88147a824c3ea826f7a1512e58b5, starting offset: 0, end offset: 9223372036854775807 2024-11-18T18:55:20,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:20,682 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:20,685 DEBUG [StoreFileCloser-94f0062296179ee1a5d0c9fc0ff8d07b-family-1 {}] bucket.BucketCache(856): removed key 4bbc88147a824c3ea826f7a1512e58b5_0 from back map with offset lock 2094080 in the evict process 2024-11-18T18:55:20,691 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T18:55:20,692 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:20,692 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Waiting for close lock at 1731956120678Running coprocessor pre-close hooks at 1731956120678Disabling compacts and flushes for region at 1731956120679 (+1 ms)Disabling writes for close at 1731956120679Writing region close event to WAL at 1731956120686 (+7 ms)Running coprocessor post-close hooks at 1731956120691 (+5 ms)Closed at 1731956120692 (+1 ms) 2024-11-18T18:55:20,692 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 94f0062296179ee1a5d0c9fc0ff8d07b move to 39fff3b0f89c,35107,1731956116307 record at close sequenceid=6 2024-11-18T18:55:20,696 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:20,696 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=CLOSED 2024-11-18T18:55:20,699 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453 because future has completed 2024-11-18T18:55:20,699 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T18:55:20,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T18:55:20,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:20,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:20,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:20,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956120682 2024-11-18T18:55:20,703 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T18:55:20,703 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T18:55:20,704 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,39895,1731956116453 in 183 msec 2024-11-18T18:55:20,704 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:20,704 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956120699 2024-11-18T18:55:20,704 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T18:55:20,705 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE; state=CLOSED, location=39fff3b0f89c,35107,1731956116307; forceNewPlan=false, retain=false 2024-11-18T18:55:20,855 INFO [39fff3b0f89c:33743 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T18:55:20,856 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=OPENING, regionLocation=39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:20,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE because future has completed 2024-11-18T18:55:20,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,35107,1731956116307}] 2024-11-18T18:55:21,020 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,021 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 94f0062296179ee1a5d0c9fc0ff8d07b, NAME => 'testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:21,022 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731956118694 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,022 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:21,023 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,023 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,024 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,025 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94f0062296179ee1a5d0c9fc0ff8d07b columnFamilyName family 2024-11-18T18:55:21,025 DEBUG [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:21,027 DEBUG [StoreFileOpener-94f0062296179ee1a5d0c9fc0ff8d07b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5, delay=938 ms 2024-11-18T18:55:21,028 DEBUG [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:21,028 INFO [StoreOpener-94f0062296179ee1a5d0c9fc0ff8d07b-1 {}] regionserver.HStore(327): Store=94f0062296179ee1a5d0c9fc0ff8d07b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:21,029 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,029 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,030 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,030 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,030 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,034 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,035 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 94f0062296179ee1a5d0c9fc0ff8d07b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67621149, jitterRate=0.007633641362190247}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:55:21,035 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,036 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Running coprocessor pre-open hook at 1731956121024Writing region info on filesystem at 1731956121024Initializing all the Stores at 1731956121024Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956121024Cleaning up temporary data from old regions at 1731956121030 (+6 ms)Running coprocessor post-open hooks at 1731956121035 (+5 ms)Region opened successfully at 1731956121036 (+1 ms) 2024-11-18T18:55:21,037 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b., pid=9, masterSystemTime=1731956121015 2024-11-18T18:55:21,040 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,040 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=94f0062296179ee1a5d0c9fc0ff8d07b, regionState=OPEN, openSeqNum=10, regionLocation=39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:21,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,35107,1731956116307 because future has completed 2024-11-18T18:55:21,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-18T18:55:21,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 94f0062296179ee1a5d0c9fc0ff8d07b, server=39fff3b0f89c,35107,1731956116307 in 184 msec 2024-11-18T18:55:21,048 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731956118694, region=94f0062296179ee1a5d0c9fc0ff8d07b, REOPEN/MOVE in 536 msec 2024-11-18T18:55:21,065 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T18:55:21,066 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:55:21,066 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:21,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,071 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T18:55:21,071 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T18:55:21,071 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-18T18:55:21,072 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=39fff3b0f89c,33743,1731956115513 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:21,114 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:21,115 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:55:21,116 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:55:21,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:21,116 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:21,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:21,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,117 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-18T18:55:21,117 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:21,117 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(3091): Received CLOSE for 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,118 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,35107,1731956116307' ***** 2024-11-18T18:55:21,118 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:55:21,118 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,39895,1731956116453' ***** 2024-11-18T18:55:21,118 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:55:21,119 INFO [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T18:55:21,119 INFO [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence 2024-11-18T18:55:21,119 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T18:55:21,119 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T18:55:21,119 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:21,120 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T18:55:21,120 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 94f0062296179ee1a5d0c9fc0ff8d07b, disabling compactions & flushes 2024-11-18T18:55:21,120 INFO [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T18:55:21,120 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,120 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,120 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T18:55:21,120 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. after waiting 0 ms 2024-11-18T18:55:21,120 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,122 DEBUG [StoreFileCloser-94f0062296179ee1a5d0c9fc0ff8d07b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/family/4bbc88147a824c3ea826f7a1512e58b5 2024-11-18T18:55:21,122 INFO [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T18:55:21,122 INFO [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bucket.persistence 2024-11-18T18:55:21,122 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T18:55:21,122 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T18:55:21,122 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T18:55:21,123 INFO [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T18:55:21,123 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T18:55:21,126 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnRegionMove1731956118694/94f0062296179ee1a5d0c9fc0ff8d07b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T18:55:21,127 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,127 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 94f0062296179ee1a5d0c9fc0ff8d07b: Waiting for close lock at 1731956121119Running coprocessor pre-close hooks at 1731956121120 (+1 ms)Disabling compacts and flushes for region at 1731956121120Disabling writes for close at 1731956121120Writing region close event to WAL at 1731956121122 (+2 ms)Running coprocessor post-close hooks at 1731956121126 (+4 ms)Closed at 1731956121126 2024-11-18T18:55:21,129 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33743 {}] assignment.AssignmentManager(1314): RegionServer CLOSED 94f0062296179ee1a5d0c9fc0ff8d07b 2024-11-18T18:55:21,129 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b. 2024-11-18T18:55:21,130 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1487): Thread RS:1;39fff3b0f89c:39895 started persisting bucket cache to file 2024-11-18T18:55:21,130 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:21,130 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:21,133 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1487): Thread RS:0;39fff3b0f89c:35107 started persisting bucket cache to file 2024-11-18T18:55:21,133 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:21,133 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:21,133 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:21,133 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956121130 2024-11-18T18:55:21,133 DEBUG [RS:1;39fff3b0f89c:39895 {}] bucket.BucketCache(1509): Thread RS:1;39fff3b0f89c:39895 finished persisting bucket cache to file, renaming 2024-11-18T18:55:21,133 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:55:21,134 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;39fff3b0f89c:39895. 2024-11-18T18:55:21,134 DEBUG [RS:1;39fff3b0f89c:39895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:21,134 DEBUG [RS:1;39fff3b0f89c:39895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:55:21,134 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T18:55:21,135 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T18:55:21,135 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T18:55:21,135 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:21,135 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:55:21,135 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:55:21,135 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:55:21,135 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:55:21,135 DEBUG [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T18:55:21,135 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:55:21,135 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-18T18:55:21,137 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:21,137 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956121133 2024-11-18T18:55:21,137 DEBUG [RS:0;39fff3b0f89c:35107 {}] bucket.BucketCache(1509): Thread RS:0;39fff3b0f89c:35107 finished persisting bucket cache to file, renaming 2024-11-18T18:55:21,137 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:55:21,137 INFO [RS:0;39fff3b0f89c:35107 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:55:21,137 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:55:21,138 INFO [RS:0;39fff3b0f89c:35107 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:55:21,138 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:21,138 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:21,138 INFO [RS:0;39fff3b0f89c:35107 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;39fff3b0f89c:35107. 2024-11-18T18:55:21,138 DEBUG [RS:0;39fff3b0f89c:35107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:21,138 DEBUG [RS:0;39fff3b0f89c:35107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,138 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,35107,1731956116307; all regions closed. 2024-11-18T18:55:21,143 DEBUG [RS:0;39fff3b0f89c:35107 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:21,143 INFO [RS:0;39fff3b0f89c:35107 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C35107%2C1731956116307:(num 1731956117954) 2024-11-18T18:55:21,143 DEBUG [RS:0;39fff3b0f89c:35107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,143 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:21,143 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:21,144 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:21,144 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:55:21,144 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:21,144 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:55:21,144 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:55:21,144 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:21,145 INFO [RS:0;39fff3b0f89c:35107 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35107 2024-11-18T18:55:21,145 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fe5f278cd0344c2b9e1e1a3398a49d45 2024-11-18T18:55:21,161 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/fe5f278cd0344c2b9e1e1a3398a49d45 is 205, key is testBlockEvictionOnRegionMove1731956118694,,1731956118709.94f0062296179ee1a5d0c9fc0ff8d07b./info:regioninfo/1731956121040/Put/seqid=0 2024-11-18T18:55:21,162 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/fe5f278cd0344c2b9e1e1a3398a49d45 2024-11-18T18:55:21,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:21,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,35107,1731956116307 2024-11-18T18:55:21,163 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:21,164 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,35107,1731956116307] 2024-11-18T18:55:21,171 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ddb53839ffd44399885975f32700fa5b 2024-11-18T18:55:21,180 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,35107,1731956116307 already deleted, retry=false 2024-11-18T18:55:21,180 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,35107,1731956116307 expired; onlineServers=1 2024-11-18T18:55:21,182 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:21,182 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:21,183 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/ddb53839ffd44399885975f32700fa5b is 43, key is default/ns:d/1731956118452/Put/seqid=0 2024-11-18T18:55:21,184 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/ddb53839ffd44399885975f32700fa5b 2024-11-18T18:55:21,192 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ca6270c0eacd438a8b2066ba4ef1690b 2024-11-18T18:55:21,203 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ca6270c0eacd438a8b2066ba4ef1690b is 78, key is testBlockEvictionOnRegionMove1731956118694/table:state/1731956119123/Put/seqid=0 2024-11-18T18:55:21,204 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ca6270c0eacd438a8b2066ba4ef1690b 2024-11-18T18:55:21,206 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/fe5f278cd0344c2b9e1e1a3398a49d45 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/fe5f278cd0344c2b9e1e1a3398a49d45 2024-11-18T18:55:21,208 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/fe5f278cd0344c2b9e1e1a3398a49d45, delay=928 ms 2024-11-18T18:55:21,209 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/fe5f278cd0344c2b9e1e1a3398a49d45, entries=15, sequenceid=15, filesize=7.6 K 2024-11-18T18:55:21,210 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/ddb53839ffd44399885975f32700fa5b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/ddb53839ffd44399885975f32700fa5b 2024-11-18T18:55:21,211 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/ddb53839ffd44399885975f32700fa5b, delay=908 ms 2024-11-18T18:55:21,212 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/ddb53839ffd44399885975f32700fa5b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-18T18:55:21,213 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ca6270c0eacd438a8b2066ba4ef1690b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ca6270c0eacd438a8b2066ba4ef1690b 2024-11-18T18:55:21,214 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ca6270c0eacd438a8b2066ba4ef1690b, delay=940 ms 2024-11-18T18:55:21,215 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ca6270c0eacd438a8b2066ba4ef1690b, entries=2, sequenceid=15, filesize=5.3 K 2024-11-18T18:55:21,216 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=15, compaction requested=false 2024-11-18T18:55:21,216 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T18:55:21,217 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/fe5f278cd0344c2b9e1e1a3398a49d45 2024-11-18T18:55:21,218 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/ddb53839ffd44399885975f32700fa5b 2024-11-18T18:55:21,218 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ca6270c0eacd438a8b2066ba4ef1690b 2024-11-18T18:55:21,222 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-18T18:55:21,222 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:55:21,223 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:21,223 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956121135Running coprocessor pre-close hooks at 1731956121135Disabling compacts and flushes for region at 1731956121135Disabling writes for close at 1731956121135Obtaining lock to block concurrent updates at 1731956121136 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731956121136Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731956121136Flushing stores of hbase:meta,,1.1588230740 at 1731956121136Flushing 1588230740/info: creating writer at 1731956121137 (+1 ms)Flushing 1588230740/info: appending metadata at 1731956121160 (+23 ms)Flushing 1588230740/info: closing flushed file at 1731956121160Flushing 1588230740/ns: creating writer at 1731956121165 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731956121182 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731956121183 (+1 ms)Flushing 1588230740/table: creating writer at 1731956121186 (+3 ms)Flushing 1588230740/table: appending metadata at 1731956121203 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731956121203Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69fad436: reopening flushed file at 1731956121206 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bb9b1ca: reopening flushed file at 1731956121209 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71274f2: reopening flushed file at 1731956121212 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 80ms, sequenceid=15, compaction requested=false at 1731956121216 (+4 ms)Writing region close event to WAL at 1731956121219 (+3 ms)Running coprocessor post-close hooks at 1731956121222 (+3 ms)Closed at 1731956121223 (+1 ms) 2024-11-18T18:55:21,223 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35107-0x1015096b9020001, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,273 INFO [RS:0;39fff3b0f89c:35107 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:21,273 INFO [RS:0;39fff3b0f89c:35107 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,35107,1731956116307; zookeeper connection closed. 2024-11-18T18:55:21,274 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@141731e5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@141731e5 2024-11-18T18:55:21,335 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,39895,1731956116453; all regions closed. 2024-11-18T18:55:21,341 DEBUG [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:21,341 INFO [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C39895%2C1731956116453.meta:.meta(num 1731956118304) 2024-11-18T18:55:21,344 DEBUG [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:21,344 INFO [RS:1;39fff3b0f89c:39895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C39895%2C1731956116453:(num 1731956117955) 2024-11-18T18:55:21,344 DEBUG [RS:1;39fff3b0f89c:39895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:21,344 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:21,344 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:21,345 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:21,345 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:21,345 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:21,346 INFO [RS:1;39fff3b0f89c:39895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39895 2024-11-18T18:55:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:21,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,39895,1731956116453 2024-11-18T18:55:21,355 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:21,356 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,39895,1731956116453] 2024-11-18T18:55:21,372 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,39895,1731956116453 already deleted, retry=false 2024-11-18T18:55:21,372 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,39895,1731956116453 expired; onlineServers=0 2024-11-18T18:55:21,372 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '39fff3b0f89c,33743,1731956115513' ***** 2024-11-18T18:55:21,372 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T18:55:21,372 INFO [M:0;39fff3b0f89c:33743 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:21,372 INFO [M:0;39fff3b0f89c:33743 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:21,372 DEBUG [M:0;39fff3b0f89c:33743 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T18:55:21,372 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T18:55:21,372 DEBUG [M:0;39fff3b0f89c:33743 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T18:55:21,372 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956117561 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956117561,5,FailOnTimeoutGroup] 2024-11-18T18:55:21,372 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956117563 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956117563,5,FailOnTimeoutGroup] 2024-11-18T18:55:21,373 INFO [M:0;39fff3b0f89c:33743 {}] hbase.ChoreService(370): Chore service for: master/39fff3b0f89c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:21,373 INFO [M:0;39fff3b0f89c:33743 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:21,373 DEBUG [M:0;39fff3b0f89c:33743 {}] master.HMaster(1795): Stopping service threads 2024-11-18T18:55:21,373 INFO [M:0;39fff3b0f89c:33743 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T18:55:21,373 INFO [M:0;39fff3b0f89c:33743 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:55:21,374 INFO [M:0;39fff3b0f89c:33743 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:55:21,374 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T18:55:21,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:21,380 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:21,380 DEBUG [M:0;39fff3b0f89c:33743 {}] zookeeper.ZKUtil(347): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T18:55:21,381 WARN [M:0;39fff3b0f89c:33743 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T18:55:21,381 INFO [M:0;39fff3b0f89c:33743 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.lastflushedseqids 2024-11-18T18:55:21,392 INFO [M:0;39fff3b0f89c:33743 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:55:21,392 INFO [M:0;39fff3b0f89c:33743 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T18:55:21,392 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:55:21,392 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:21,392 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:21,392 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:55:21,392 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:21,393 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-18T18:55:21,402 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5fca0d95e3fc4827992393152c4b374a 2024-11-18T18:55:21,415 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fca0d95e3fc4827992393152c4b374a is 82, key is hbase:meta,,1/info:regioninfo/1731956118377/Put/seqid=0 2024-11-18T18:55:21,416 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fca0d95e3fc4827992393152c4b374a 2024-11-18T18:55:21,423 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bb677b59771d4519a4d8ca94a3b15690 2024-11-18T18:55:21,437 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb677b59771d4519a4d8ca94a3b15690 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731956119129/Put/seqid=0 2024-11-18T18:55:21,438 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb677b59771d4519a4d8ca94a3b15690 2024-11-18T18:55:21,445 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8504b6d01b59410c85a8b1ff91a2c17f 2024-11-18T18:55:21,457 DEBUG [M:0;39fff3b0f89c:33743 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8504b6d01b59410c85a8b1ff91a2c17f is 69, key is 39fff3b0f89c,35107,1731956116307/rs:state/1731956117680/Put/seqid=0 2024-11-18T18:55:21,457 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8504b6d01b59410c85a8b1ff91a2c17f 2024-11-18T18:55:21,459 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5fca0d95e3fc4827992393152c4b374a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5fca0d95e3fc4827992393152c4b374a 2024-11-18T18:55:21,461 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5fca0d95e3fc4827992393152c4b374a, entries=8, sequenceid=84, filesize=5.5 K 2024-11-18T18:55:21,461 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/bb677b59771d4519a4d8ca94a3b15690 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb677b59771d4519a4d8ca94a3b15690 2024-11-18T18:55:21,463 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/bb677b59771d4519a4d8ca94a3b15690, entries=9, sequenceid=84, filesize=6.5 K 2024-11-18T18:55:21,463 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8504b6d01b59410c85a8b1ff91a2c17f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8504b6d01b59410c85a8b1ff91a2c17f 2024-11-18T18:55:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,464 INFO [RS:1;39fff3b0f89c:39895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:21,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39895-0x1015096b9020002, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,464 INFO [RS:1;39fff3b0f89c:39895 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,39895,1731956116453; zookeeper connection closed. 2024-11-18T18:55:21,464 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2a4276a0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2a4276a0 2024-11-18T18:55:21,464 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-18T18:55:21,465 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8504b6d01b59410c85a8b1ff91a2c17f, entries=2, sequenceid=84, filesize=5.1 K 2024-11-18T18:55:21,466 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=84, compaction requested=false 2024-11-18T18:55:21,467 INFO [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:21,467 DEBUG [M:0;39fff3b0f89c:33743 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956121392Disabling compacts and flushes for region at 1731956121392Disabling writes for close at 1731956121392Obtaining lock to block concurrent updates at 1731956121393 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731956121393Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731956121393Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731956121394 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731956121394Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731956121414 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731956121414Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731956121417 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731956121436 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731956121436Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731956121440 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731956121456 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731956121456Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ea17dca: reopening flushed file at 1731956121459 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@581030e1: reopening flushed file at 1731956121461 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e309061: reopening flushed file at 1731956121463 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=84, compaction requested=false at 1731956121466 (+3 ms)Writing region close event to WAL at 1731956121467 (+1 ms)Closed at 1731956121467 2024-11-18T18:55:21,469 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:21,469 INFO [M:0;39fff3b0f89c:33743 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T18:55:21,469 INFO [M:0;39fff3b0f89c:33743 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33743 2024-11-18T18:55:21,469 INFO [M:0;39fff3b0f89c:33743 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:21,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,580 INFO [M:0;39fff3b0f89c:33743 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:21,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33743-0x1015096b9020000, quorum=127.0.0.1:56408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:21,589 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T18:55:21,610 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T18:55:21,611 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:21,618 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/39fff3b0f89c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731956121211 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731956119485 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731956121027 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/39fff3b0f89c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/39fff3b0f89c:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731956121208 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/39fff3b0f89c:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=103 (was 77) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5171 (was 5514) 2024-11-18T18:55:21,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=103, ProcessCount=11, AvailableMemoryMB=5170 2024-11-18T18:55:21,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/hadoop-log-dir so I do NOT create it in target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d 2024-11-18T18:55:21,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/hadoop-tmp-dir so I do NOT create it in target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d 2024-11-18T18:55:21,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:21,627 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/cluster_c96e5d20-7b9b-8d70-070b-1cbb82bbcb4b, deleteOnExit=true 2024-11-18T18:55:21,630 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/cluster_c96e5d20-7b9b-8d70-070b-1cbb82bbcb4b/zookeeper_0, clientPort=54605, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/cluster_c96e5d20-7b9b-8d70-070b-1cbb82bbcb4b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/cluster_c96e5d20-7b9b-8d70-070b-1cbb82bbcb4b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T18:55:21,632 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54605 2024-11-18T18:55:21,643 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 with version=8 2024-11-18T18:55:21,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase-staging 2024-11-18T18:55:21,646 INFO [Time-limited test {}] client.ConnectionUtils(128): master/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:21,646 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T18:55:21,647 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:21,648 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38159 2024-11-18T18:55:21,650 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38159 connecting to ZooKeeper ensemble=127.0.0.1:54605 2024-11-18T18:55:21,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:381590x0, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:21,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38159-0x1015096d2060000 connected 2024-11-18T18:55:21,760 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:21,761 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:21,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:21,764 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26, hbase.cluster.distributed=false 2024-11-18T18:55:21,770 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:21,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38159 2024-11-18T18:55:21,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38159 2024-11-18T18:55:21,771 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38159 2024-11-18T18:55:21,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38159 2024-11-18T18:55:21,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38159 2024-11-18T18:55:21,790 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:21,790 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:55:21,791 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:21,791 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45879 2024-11-18T18:55:21,793 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45879 connecting to ZooKeeper ensemble=127.0.0.1:54605 2024-11-18T18:55:21,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:21,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:22,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458790x0, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:22,008 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45879-0x1015096d2060001 connected 2024-11-18T18:55:22,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:22,010 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:55:22,012 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache 2024-11-18T18:55:22,013 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T18:55:22,013 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T18:55:22,014 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T18:55:22,014 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T18:55:22,014 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T18:55:22,015 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T18:55:22,015 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:55:22,016 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T18:55:22,016 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence is complete. 2024-11-18T18:55:22,016 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:55:22,017 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:22,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45879 2024-11-18T18:55:22,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45879 2024-11-18T18:55:22,018 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45879 2024-11-18T18:55:22,019 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45879 2024-11-18T18:55:22,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45879 2024-11-18T18:55:22,036 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:55:22,036 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:22,037 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46121 2024-11-18T18:55:22,038 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46121 connecting to ZooKeeper ensemble=127.0.0.1:54605 2024-11-18T18:55:22,038 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:22,039 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:22,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461210x0, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:22,058 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46121-0x1015096d2060002 connected 2024-11-18T18:55:22,058 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:22,059 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:55:22,060 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache 2024-11-18T18:55:22,060 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T18:55:22,060 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T18:55:22,060 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T18:55:22,061 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T18:55:22,061 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T18:55:22,061 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T18:55:22,061 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T18:55:22,061 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence is complete. 2024-11-18T18:55:22,061 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:55:22,062 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T18:55:22,064 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:22,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-18T18:55:22,065 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46121 2024-11-18T18:55:22,066 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46121 2024-11-18T18:55:22,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-18T18:55:22,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46121 2024-11-18T18:55:22,084 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;39fff3b0f89c:38159 2024-11-18T18:55:22,084 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,093 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,101 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,101 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T18:55:22,102 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/39fff3b0f89c,38159,1731956121646 from backup master directory 2024-11-18T18:55:22,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T18:55:22,109 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:22,109 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,110 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase.id] with ID: 1bd3858c-12c7-4b95-b968-c2117b040b03 2024-11-18T18:55:22,110 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.tmp/hbase.id 2024-11-18T18:55:22,114 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/hbase.id] 2024-11-18T18:55:22,145 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:22,145 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T18:55:22,147 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-18T18:55:22,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,163 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:55:22,165 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T18:55:22,165 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:22,184 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store 2024-11-18T18:55:22,190 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:22,190 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:55:22,190 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:22,190 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:22,190 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:55:22,190 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:22,191 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:22,191 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956122190Disabling compacts and flushes for region at 1731956122190Disabling writes for close at 1731956122190Writing region close event to WAL at 1731956122190Closed at 1731956122190 2024-11-18T18:55:22,191 WARN [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/.initializing 2024-11-18T18:55:22,191 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,196 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:22,200 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=39fff3b0f89c%2C38159%2C1731956121646, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,38159,1731956121646, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/oldWALs, maxLogs=10 2024-11-18T18:55:22,216 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/WALs/39fff3b0f89c,38159,1731956121646/39fff3b0f89c%2C38159%2C1731956121646.1731956122200 2024-11-18T18:55:22,217 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:22,217 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:22,217 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:22,217 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,217 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T18:55:22,222 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:22,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T18:55:22,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:22,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T18:55:22,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:22,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T18:55:22,233 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,233 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:22,234 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,234 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,234 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,235 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,235 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,235 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:22,236 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T18:55:22,242 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:22,242 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64838248, jitterRate=-0.03383481502532959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:22,244 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731956122217Initializing all the Stores at 1731956122217Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956122218 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956122218Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956122218Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956122218Cleaning up temporary data from old regions at 1731956122235 (+17 ms)Region opened successfully at 1731956122243 (+8 ms) 2024-11-18T18:55:22,244 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T18:55:22,247 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f7276c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:22,249 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T18:55:22,249 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T18:55:22,249 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T18:55:22,249 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T18:55:22,249 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T18:55:22,250 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-18T18:55:22,250 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T18:55:22,252 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T18:55:22,253 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T18:55:22,259 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T18:55:22,259 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T18:55:22,260 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T18:55:22,267 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T18:55:22,268 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T18:55:22,269 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T18:55:22,276 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T18:55:22,277 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T18:55:22,284 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T18:55:22,287 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T18:55:22,292 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,303 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=39fff3b0f89c,38159,1731956121646, sessionid=0x1015096d2060000, setting cluster-up flag (Was=false) 2024-11-18T18:55:22,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,342 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T18:55:22,344 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,384 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T18:55:22,385 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:22,386 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T18:55:22,388 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:22,388 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T18:55:22,388 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T18:55:22,389 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 39fff3b0f89c,38159,1731956121646 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=5, maxPoolSize=5 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/39fff3b0f89c:0, corePoolSize=10, maxPoolSize=10 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:22,391 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731956152392 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T18:55:22,392 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:22,393 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T18:55:22,393 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T18:55:22,393 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,394 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956122393,5,FailOnTimeoutGroup] 2024-11-18T18:55:22,394 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956122394,5,FailOnTimeoutGroup] 2024-11-18T18:55:22,394 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,394 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T18:55:22,394 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T18:55:22,394 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,394 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,413 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T18:55:22,413 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:22,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:22,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T18:55:22,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:55:22,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:22,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T18:55:22,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:55:22,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,426 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:22,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T18:55:22,429 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:55:22,429 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:22,430 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T18:55:22,432 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:55:22,432 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:22,433 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:22,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:55:22,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:22,433 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:22,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:55:22,434 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:55:22,434 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:22,435 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:55:22,440 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:22,441 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61068894, jitterRate=-0.09000256657600403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:22,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731956122419Initializing all the Stores at 1731956122420 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956122420Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956122420Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956122420Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956122420Cleaning up temporary data from old regions at 1731956122434 (+14 ms)Region opened successfully at 1731956122442 (+8 ms) 2024-11-18T18:55:22,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:55:22,442 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:55:22,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:55:22,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:55:22,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:55:22,443 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:22,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956122442Disabling compacts and flushes for region at 1731956122442Disabling writes for close at 1731956122442Writing region close event to WAL at 1731956122443 (+1 ms)Closed at 1731956122443 2024-11-18T18:55:22,444 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:22,444 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T18:55:22,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:55:22,446 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:22,446 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T18:55:22,473 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(746): ClusterId : 1bd3858c-12c7-4b95-b968-c2117b040b03 2024-11-18T18:55:22,474 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(746): ClusterId : 1bd3858c-12c7-4b95-b968-c2117b040b03 2024-11-18T18:55:22,474 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:55:22,474 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:55:22,485 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:55:22,485 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:55:22,485 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:55:22,485 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:55:22,493 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:55:22,493 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:55:22,494 DEBUG [RS:1;39fff3b0f89c:46121 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20ebe536, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:22,494 DEBUG [RS:0;39fff3b0f89c:45879 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd0d5b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:22,505 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;39fff3b0f89c:45879 2024-11-18T18:55:22,505 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;39fff3b0f89c:46121 2024-11-18T18:55:22,506 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:55:22,506 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:55:22,506 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:55:22,506 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:55:22,506 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:55:22,506 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:55:22,507 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,38159,1731956121646 with port=46121, startcode=1731956122035 2024-11-18T18:55:22,507 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,38159,1731956121646 with port=45879, startcode=1731956121790 2024-11-18T18:55:22,507 DEBUG [RS:1;39fff3b0f89c:46121 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:55:22,507 DEBUG [RS:0;39fff3b0f89c:45879 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:55:22,509 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52517, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:55:22,509 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:55:22,510 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,510 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,511 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,512 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:22,512 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T18:55:22,512 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:55:22,513 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:22,513 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T18:55:22,513 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:55:22,517 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:22,550 DEBUG [RS:0;39fff3b0f89c:45879 {}] zookeeper.ZKUtil(111): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,550 WARN [RS:0;39fff3b0f89c:45879 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:22,550 INFO [RS:0;39fff3b0f89c:45879 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:22,550 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,45879,1731956121790] 2024-11-18T18:55:22,550 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,550 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,46121,1731956122035] 2024-11-18T18:55:22,550 DEBUG [RS:1;39fff3b0f89c:46121 {}] zookeeper.ZKUtil(111): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,551 WARN [RS:1;39fff3b0f89c:46121 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:22,551 INFO [RS:1;39fff3b0f89c:46121 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:22,551 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,558 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:55:22,558 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:55:22,561 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:55:22,562 INFO [RS:0;39fff3b0f89c:45879 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:55:22,562 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,562 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:55:22,563 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:55:22,563 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,563 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:55:22,563 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,563 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,563 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,564 INFO [RS:1;39fff3b0f89c:46121 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:55:22,564 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,564 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,564 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,564 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:22,564 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:55:22,564 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,564 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:22,565 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,565 DEBUG [RS:0;39fff3b0f89c:45879 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:22,565 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,566 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,567 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,567 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,567 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:22,567 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:22,567 DEBUG [RS:1;39fff3b0f89c:46121 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:22,567 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,567 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,567 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,567 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,567 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,568 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,45879,1731956121790-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,569 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,46121,1731956122035-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:22,589 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:55:22,589 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:55:22,589 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,45879,1731956121790-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,589 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,46121,1731956122035-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,590 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,590 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,590 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.Replication(171): 39fff3b0f89c,45879,1731956121790 started 2024-11-18T18:55:22,590 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.Replication(171): 39fff3b0f89c,46121,1731956122035 started 2024-11-18T18:55:22,597 WARN [39fff3b0f89c:38159 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T18:55:22,606 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,606 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:22,606 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,45879,1731956121790, RpcServer on 39fff3b0f89c/172.17.0.2:45879, sessionid=0x1015096d2060001 2024-11-18T18:55:22,606 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,46121,1731956122035, RpcServer on 39fff3b0f89c/172.17.0.2:46121, sessionid=0x1015096d2060002 2024-11-18T18:55:22,606 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:55:22,606 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:55:22,606 DEBUG [RS:1;39fff3b0f89c:46121 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,606 DEBUG [RS:0;39fff3b0f89c:45879 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,606 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,46121,1731956122035' 2024-11-18T18:55:22,606 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,45879,1731956121790' 2024-11-18T18:55:22,606 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:55:22,606 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:55:22,607 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:55:22,607 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,46121,1731956122035' 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,45879,1731956121790' 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:55:22,608 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:55:22,608 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:55:22,609 DEBUG [RS:1;39fff3b0f89c:46121 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:55:22,609 DEBUG [RS:0;39fff3b0f89c:45879 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:55:22,609 INFO [RS:1;39fff3b0f89c:46121 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:55:22,609 INFO [RS:0;39fff3b0f89c:45879 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:55:22,609 INFO [RS:1;39fff3b0f89c:46121 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:55:22,609 INFO [RS:0;39fff3b0f89c:45879 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:55:22,710 INFO [RS:0;39fff3b0f89c:45879 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:22,710 INFO [RS:1;39fff3b0f89c:46121 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:22,712 INFO [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C45879%2C1731956121790, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:22,713 INFO [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C46121%2C1731956122035, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,46121,1731956122035, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:22,733 INFO [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790/39fff3b0f89c%2C45879%2C1731956121790.1731956122717 2024-11-18T18:55:22,733 INFO [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,46121,1731956122035/39fff3b0f89c%2C46121%2C1731956122035.1731956122717 2024-11-18T18:55:22,733 DEBUG [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:22,733 DEBUG [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:22,847 DEBUG [39fff3b0f89c:38159 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T18:55:22,847 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T18:55:22,848 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:55:22,848 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:55:22,848 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:55:22,848 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:55:22,849 INFO [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:55:22,849 INFO [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:55:22,849 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:55:22,849 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:22,850 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,45879,1731956121790, state=OPENING 2024-11-18T18:55:22,867 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T18:55:22,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:22,877 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:22,877 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:22,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:22,878 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:22,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,45879,1731956121790}] 2024-11-18T18:55:23,033 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:55:23,037 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60893, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:55:23,041 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:55:23,041 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:23,042 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:55:23,044 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C45879%2C1731956121790.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:23,061 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790/39fff3b0f89c%2C45879%2C1731956121790.meta.1731956123045.meta 2024-11-18T18:55:23,062 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:23,062 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:23,062 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:55:23,062 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:55:23,062 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:55:23,062 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:55:23,063 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:23,063 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:55:23,063 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:55:23,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T18:55:23,064 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:55:23,064 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:23,064 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T18:55:23,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:55:23,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:23,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T18:55:23,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:55:23,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:23,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T18:55:23,066 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:55:23,067 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:23,067 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:55:23,068 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:23,069 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:23,069 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:55:23,069 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:55:23,069 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:23,071 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:55:23,071 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61309021, jitterRate=-0.08642439544200897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:23,072 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:55:23,072 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731956123063Writing region info on filesystem at 1731956123063Initializing all the Stores at 1731956123063Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956123063Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956123063Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956123063Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956123063Cleaning up temporary data from old regions at 1731956123069 (+6 ms)Running coprocessor post-open hooks at 1731956123072 (+3 ms)Region opened successfully at 1731956123072 2024-11-18T18:55:23,073 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731956123032 2024-11-18T18:55:23,076 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:55:23,076 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:55:23,077 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:23,077 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,45879,1731956121790, state=OPEN 2024-11-18T18:55:23,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:23,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:23,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:23,084 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:23,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:23,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:23,084 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:23,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T18:55:23,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,45879,1731956121790 in 206 msec 2024-11-18T18:55:23,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T18:55:23,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 642 msec 2024-11-18T18:55:23,089 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T18:55:23,089 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T18:55:23,093 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:55:23,093 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1] 2024-11-18T18:55:23,094 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:55:23,096 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49897, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:55:23,103 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 715 msec 2024-11-18T18:55:23,103 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731956123103, completionTime=-1 2024-11-18T18:55:23,103 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T18:55:23,103 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T18:55:23,106 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T18:55:23,106 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731956183106 2024-11-18T18:55:23,106 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731956243106 2024-11-18T18:55:23,106 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-18T18:55:23,106 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,107 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,107 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,107 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-39fff3b0f89c:38159, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,107 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,107 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,110 DEBUG [master/39fff3b0f89c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T18:55:23,112 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.003sec 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:23,113 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T18:55:23,114 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T18:55:23,114 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T18:55:23,114 INFO [master/39fff3b0f89c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,38159,1731956121646-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:23,173 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:23,173 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 39fff3b0f89c,38159,-1 for getting cluster id 2024-11-18T18:55:23,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T18:55:23,175 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1bd3858c-12c7-4b95-b968-c2117b040b03' 2024-11-18T18:55:23,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T18:55:23,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1bd3858c-12c7-4b95-b968-c2117b040b03" 2024-11-18T18:55:23,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ccabd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:23,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [39fff3b0f89c,38159,-1] 2024-11-18T18:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T18:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:23,179 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T18:55:23,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T18:55:23,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:55:23,183 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1] 2024-11-18T18:55:23,184 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:55:23,187 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:55:23,190 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:23,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T18:55:23,192 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:23,192 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27133292 2024-11-18T18:55:23,193 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T18:55:23,195 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36844, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T18:55:23,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731956123190', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T18:55:23,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 2024-11-18T18:55:23,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T18:55:23,200 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731956123190" procId is: 4 2024-11-18T18:55:23,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T18:55:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:23,211 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => db9393bb5bef5ac659bddcd971022f24, NAME => 'testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731956123190', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:23,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:23,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1722): Closing db9393bb5bef5ac659bddcd971022f24, disabling compactions & flushes 2024-11-18T18:55:23,218 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. after waiting 0 ms 2024-11-18T18:55:23,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,219 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,219 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731956123190-pool-0 {}] regionserver.HRegion(1676): Region close journal for db9393bb5bef5ac659bddcd971022f24: Waiting for close lock at 1731956123218Disabling compacts and flushes for region at 1731956123218Disabling writes for close at 1731956123218Writing region close event to WAL at 1731956123219 (+1 ms)Closed at 1731956123219 2024-11-18T18:55:23,220 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T18:55:23,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731956123220"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731956123220"}]},"ts":"1731956123220"} 2024-11-18T18:55:23,223 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T18:55:23,224 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T18:55:23,225 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731956123190","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956123224"}]},"ts":"1731956123224"} 2024-11-18T18:55:23,227 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731956123190, state=ENABLING in hbase:meta 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:55:23,228 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:55:23,228 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:55:23,228 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:55:23,229 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN}] 2024-11-18T18:55:23,230 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN 2024-11-18T18:55:23,231 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN; state=OFFLINE, location=39fff3b0f89c,45879,1731956121790; forceNewPlan=false, retain=false 2024-11-18T18:55:23,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:23,382 INFO [39fff3b0f89c:38159 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T18:55:23,383 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=db9393bb5bef5ac659bddcd971022f24, regionState=OPENING, regionLocation=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:23,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN because future has completed 2024-11-18T18:55:23,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,45879,1731956121790}] 2024-11-18T18:55:23,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:23,547 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,547 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => db9393bb5bef5ac659bddcd971022f24, NAME => 'testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:23,548 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731956123190 db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,548 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:23,548 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,549 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,550 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,557 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db9393bb5bef5ac659bddcd971022f24 columnFamilyName family 2024-11-18T18:55:23,557 DEBUG [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:23,557 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] regionserver.HStore(327): Store=db9393bb5bef5ac659bddcd971022f24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:23,558 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,559 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,559 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,559 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,559 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,561 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,567 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T18:55:23,568 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened db9393bb5bef5ac659bddcd971022f24; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72253502, jitterRate=0.07666108012199402}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:55:23,568 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:23,568 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for db9393bb5bef5ac659bddcd971022f24: Running coprocessor pre-open hook at 1731956123549Writing region info on filesystem at 1731956123549Initializing all the Stores at 1731956123549Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956123549Cleaning up temporary data from old regions at 1731956123559 (+10 ms)Running coprocessor post-open hooks at 1731956123568 (+9 ms)Region opened successfully at 1731956123568 2024-11-18T18:55:23,569 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24., pid=6, masterSystemTime=1731956123543 2024-11-18T18:55:23,572 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,572 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:23,572 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=db9393bb5bef5ac659bddcd971022f24, regionState=OPEN, openSeqNum=2, regionLocation=39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:23,575 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,45879,1731956121790 because future has completed 2024-11-18T18:55:23,578 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T18:55:23,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,45879,1731956121790 in 188 msec 2024-11-18T18:55:23,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T18:55:23,581 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN in 351 msec 2024-11-18T18:55:23,581 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T18:55:23,582 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731956123190","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731956123582"}]},"ts":"1731956123582"} 2024-11-18T18:55:23,584 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731956123190, state=ENABLED in hbase:meta 2024-11-18T18:55:23,585 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T18:55:23,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731956123190 in 388 msec 2024-11-18T18:55:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38159 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T18:55:23,830 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731956123190 completed 2024-11-18T18:55:23,830 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731956123190 get assigned. Timeout = 60000ms 2024-11-18T18:55:23,830 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:55:23,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731956123190 assigned to meta. Checking AM states. 2024-11-18T18:55:23,835 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T18:55:23,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731956123190 assigned. 2024-11-18T18:55:23,840 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731956123190', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24., hostname=39fff3b0f89c,45879,1731956121790, seqNum=2] 2024-11-18T18:55:23,847 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing db9393bb5bef5ac659bddcd971022f24 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T18:55:23,847 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T18:55:23,856 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 593875e25e044aff94057864221682ec 2024-11-18T18:55:23,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/.tmp/family/593875e25e044aff94057864221682ec is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T18:55:23,873 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/.tmp/family/593875e25e044aff94057864221682ec 2024-11-18T18:55:23,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/.tmp/family/593875e25e044aff94057864221682ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec 2024-11-18T18:55:23,877 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, delay=948 ms 2024-11-18T18:55:23,878 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T18:55:23,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for db9393bb5bef5ac659bddcd971022f24 in 32ms, sequenceid=6, compaction requested=false 2024-11-18T18:55:23,879 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731956123190' 2024-11-18T18:55:23,880 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for db9393bb5bef5ac659bddcd971022f24: 2024-11-18T18:55:24,827 DEBUG [hfile-prefetch-1731956121027 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 593875e25e044aff94057864221682ec_0. 2024-11-18T18:55:24,828 DEBUG [hfile-prefetch-1731956121027 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 593875e25e044aff94057864221682ec_117. 2024-11-18T18:55:24,829 DEBUG [hfile-prefetch-1731956121027 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, with total blocks 2, and data blocks 1 2024-11-18T18:55:24,829 DEBUG [hfile-prefetch-1731956121027 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T18:55:24,829 DEBUG [hfile-prefetch-1731956121027 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T18:55:24,829 DEBUG [hfile-prefetch-1731956121027 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec. Will sleep 100ms and try the verification again. 2024-11-18T18:55:24,880 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;39fff3b0f89c:45879,5,FailOnTimeoutGroup] 2024-11-18T18:55:24,881 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,45879,1731956121790' ***** 2024-11-18T18:55:24,881 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-18T18:55:24,882 INFO [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T18:55:24,909 INFO [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence 2024-11-18T18:55:24,909 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T18:55:24,909 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T18:55:24,909 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T18:55:24,909 INFO [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T18:55:24,909 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T18:55:24,919 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1487): Thread RS:0;39fff3b0f89c:45879 started persisting bucket cache to file 2024-11-18T18:55:24,919 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:24,919 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:24,925 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T18:55:24,925 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731956124919 2024-11-18T18:55:24,925 DEBUG [RS:0;39fff3b0f89c:45879 {}] bucket.BucketCache(1509): Thread RS:0;39fff3b0f89c:45879 finished persisting bucket cache to file, renaming 2024-11-18T18:55:24,925 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:55:24,925 INFO [RS:0;39fff3b0f89c:45879 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:55:24,925 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:55:24,925 INFO [RS:0;39fff3b0f89c:45879 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(3091): Received CLOSE for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;39fff3b0f89c:45879. 2024-11-18T18:55:24,926 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:24,926 DEBUG [RS:0;39fff3b0f89c:45879 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:24,926 DEBUG [RS:0;39fff3b0f89c:45879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:24,926 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db9393bb5bef5ac659bddcd971022f24, disabling compactions & flushes 2024-11-18T18:55:24,926 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:55:24,926 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:55:24,926 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. after waiting 0 ms 2024-11-18T18:55:24,926 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:24,926 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T18:55:24,927 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T18:55:24,927 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1325): Online Regions={db9393bb5bef5ac659bddcd971022f24=testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24., 1588230740=hbase:meta,,1.1588230740} 2024-11-18T18:55:24,927 DEBUG [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:24,927 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:24,927 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:55:24,927 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:55:24,927 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:55:24,927 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:55:24,927 DEBUG [StoreFileCloser-db9393bb5bef5ac659bddcd971022f24-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec 2024-11-18T18:55:24,927 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:55:24,927 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-18T18:55:24,927 WARN [hfile-prefetch-1731956121027 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T18:55:24,928 DEBUG [hfile-prefetch-1731956121027 {}] hfile.PrefetchExecutor(121): Prefetch completed for 593875e25e044aff94057864221682ec 2024-11-18T18:55:24,931 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T18:55:24,932 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:24,932 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db9393bb5bef5ac659bddcd971022f24: Waiting for close lock at 1731956124926Running coprocessor pre-close hooks at 1731956124926Disabling compacts and flushes for region at 1731956124926Disabling writes for close at 1731956124926Writing region close event to WAL at 1731956124927 (+1 ms)Running coprocessor post-close hooks at 1731956124932 (+5 ms)Closed at 1731956124932 2024-11-18T18:55:24,932 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:24,936 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:24,950 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/eab3e71fa4fe48aba9d27df7c4ae7438 is 209, key is testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24./info:regioninfo/1731956123572/Put/seqid=0 2024-11-18T18:55:24,951 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:24,958 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:24,963 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T18:55:24,963 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T18:55:24,964 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:24,964 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:24,966 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:24,966 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956124964 2024-11-18T18:55:24,966 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T18:55:24,971 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:24,972 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/49e35eb3bf16499eb1514b6b74a38508 is 43, key is default/ns:d/1731956123097/Put/seqid=0 2024-11-18T18:55:24,972 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:24,980 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:24,992 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ed788b7b90334c88b97d60fb643513cc is 80, key is testBlockEvictionOnGracefulStop1731956123190/table:state/1731956123582/Put/seqid=0 2024-11-18T18:55:24,993 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:24,995 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/eab3e71fa4fe48aba9d27df7c4ae7438 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:24,996 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438, delay=923 ms 2024-11-18T18:55:24,997 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438, entries=10, sequenceid=11, filesize=7.0 K 2024-11-18T18:55:24,998 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/ns/49e35eb3bf16499eb1514b6b74a38508 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:24,999 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508, delay=916 ms 2024-11-18T18:55:25,000 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508, entries=2, sequenceid=11, filesize=5.0 K 2024-11-18T18:55:25,000 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/table/ed788b7b90334c88b97d60fb643513cc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:25,001 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc, delay=982 ms 2024-11-18T18:55:25,002 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc, entries=2, sequenceid=11, filesize=5.3 K 2024-11-18T18:55:25,003 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false 2024-11-18T18:55:25,004 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:25,004 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:25,005 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:25,009 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-18T18:55:25,010 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:55:25,010 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:25,010 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956124927Running coprocessor pre-close hooks at 1731956124927Disabling compacts and flushes for region at 1731956124927Disabling writes for close at 1731956124927Obtaining lock to block concurrent updates at 1731956124927Preparing flush snapshotting stores in 1588230740 at 1731956124927Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731956124928 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731956124928Flushing 1588230740/info: creating writer at 1731956124928Flushing 1588230740/info: appending metadata at 1731956124950 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731956124950Flushing 1588230740/ns: creating writer at 1731956124953 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731956124971 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731956124971Flushing 1588230740/table: creating writer at 1731956124974 (+3 ms)Flushing 1588230740/table: appending metadata at 1731956124991 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731956124992 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cd776a0: reopening flushed file at 1731956124995 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4278feb1: reopening flushed file at 1731956124998 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27809f3d: reopening flushed file at 1731956125000 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=11, compaction requested=false at 1731956125003 (+3 ms)Writing region close event to WAL at 1731956125006 (+3 ms)Running coprocessor post-close hooks at 1731956125010 (+4 ms)Closed at 1731956125010 2024-11-18T18:55:25,010 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:25,127 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,45879,1731956121790; all regions closed. 2024-11-18T18:55:25,133 DEBUG [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:25,134 INFO [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C45879%2C1731956121790.meta:.meta(num 1731956123045) 2024-11-18T18:55:25,139 DEBUG [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:25,139 INFO [RS:0;39fff3b0f89c:45879 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C45879%2C1731956121790:(num 1731956122717) 2024-11-18T18:55:25,139 DEBUG [RS:0;39fff3b0f89c:45879 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:25,139 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:25,139 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:25,139 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:25,139 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:25,140 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:25,140 INFO [RS:0;39fff3b0f89c:45879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45879 2024-11-18T18:55:25,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,45879,1731956121790 2024-11-18T18:55:25,150 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:25,151 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:25,159 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,45879,1731956121790] 2024-11-18T18:55:25,167 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,45879,1731956121790 already deleted, retry=false 2024-11-18T18:55:25,167 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 39fff3b0f89c,45879,1731956121790 on 39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:25,174 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 39fff3b0f89c,45879,1731956121790, splitWal=true, meta=true 2024-11-18T18:55:25,176 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 39fff3b0f89c,45879,1731956121790 (carryingMeta=true) 39fff3b0f89c,45879,1731956121790/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@33123c2a[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T18:55:25,176 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 39fff3b0f89c,45879,1731956121790, splitWal=true, meta=true 2024-11-18T18:55:25,178 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 39fff3b0f89c,45879,1731956121790, splitWal=true, meta=true, isMeta: true 2024-11-18T18:55:25,178 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790-splitting 2024-11-18T18:55:25,178 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790-splitting dir is empty, no logs to split. 2024-11-18T18:55:25,178 INFO [PEWorker-2 {}] master.SplitWALManager(105): 39fff3b0f89c,45879,1731956121790 WAL count=0, meta=true 2024-11-18T18:55:25,181 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790-splitting dir is empty, no logs to split. 2024-11-18T18:55:25,181 INFO [PEWorker-2 {}] master.SplitWALManager(105): 39fff3b0f89c,45879,1731956121790 WAL count=0, meta=true 2024-11-18T18:55:25,181 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 39fff3b0f89c,45879,1731956121790 WAL splitting is done? wals=0, meta=true 2024-11-18T18:55:25,183 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T18:55:25,184 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:25,185 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:55:25,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:25,259 INFO [RS:0;39fff3b0f89c:45879 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:25,259 INFO [RS:0;39fff3b0f89c:45879 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,45879,1731956121790; zookeeper connection closed. 2024-11-18T18:55:25,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45879-0x1015096d2060001, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:25,259 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@47235086 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@47235086 2024-11-18T18:55:25,335 DEBUG [39fff3b0f89c:38159 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-18T18:55:25,336 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:25,337 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,46121,1731956122035, state=OPENING 2024-11-18T18:55:25,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:25,342 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:25,343 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T18:55:25,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:25,343 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:25,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,46121,1731956122035}] 2024-11-18T18:55:25,404 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/39fff3b0f89c:0 server-side Connection retries=45 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T18:55:25,404 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T18:55:25,405 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41543 2024-11-18T18:55:25,406 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41543 connecting to ZooKeeper ensemble=127.0.0.1:54605 2024-11-18T18:55:25,407 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:25,407 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:25,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415430x0, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T18:55:25,418 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-18T18:55:25,418 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-18T18:55:25,418 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41543-0x1015096d2060003 connected 2024-11-18T18:55:25,419 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T18:55:25,420 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache 2024-11-18T18:55:25,420 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T18:55:25,420 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T18:55:25,420 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T18:55:25,420 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T18:55:25,420 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T18:55:25,420 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T18:55:25,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T18:55:25,421 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-18T18:55:25,422 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-18T18:55:25,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T18:55:25,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-18T18:55:25,427 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41543 2024-11-18T18:55:25,427 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-18T18:55:25,428 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T18:55:25,428 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-18T18:55:25,428 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence is complete. 2024-11-18T18:55:25,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41543 2024-11-18T18:55:25,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-18T18:55:25,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41543 2024-11-18T18:55:25,436 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(746): ClusterId : 1bd3858c-12c7-4b95-b968-c2117b040b03 2024-11-18T18:55:25,436 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T18:55:25,443 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T18:55:25,443 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T18:55:25,451 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T18:55:25,452 DEBUG [RS:2;39fff3b0f89c:41543 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22db2749, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=39fff3b0f89c/172.17.0.2:0 2024-11-18T18:55:25,462 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;39fff3b0f89c:41543 2024-11-18T18:55:25,462 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T18:55:25,462 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T18:55:25,462 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T18:55:25,463 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(2659): reportForDuty to master=39fff3b0f89c,38159,1731956121646 with port=41543, startcode=1731956125403 2024-11-18T18:55:25,464 DEBUG [RS:2;39fff3b0f89c:41543 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T18:55:25,466 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44887, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T18:55:25,466 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,467 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38159 {}] master.ServerManager(517): Registering regionserver=39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,468 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26 2024-11-18T18:55:25,468 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T18:55:25,468 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T18:55:25,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:25,476 DEBUG [RS:2;39fff3b0f89c:41543 {}] zookeeper.ZKUtil(111): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,476 WARN [RS:2;39fff3b0f89c:41543 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T18:55:25,476 INFO [RS:2;39fff3b0f89c:41543 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:25,477 DEBUG [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,477 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [39fff3b0f89c,41543,1731956125403] 2024-11-18T18:55:25,481 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T18:55:25,483 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T18:55:25,484 INFO [RS:2;39fff3b0f89c:41543 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T18:55:25,484 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,484 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T18:55:25,484 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T18:55:25,484 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,484 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/39fff3b0f89c:0, corePoolSize=2, maxPoolSize=2 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/39fff3b0f89c:0, corePoolSize=1, maxPoolSize=1 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:25,485 DEBUG [RS:2;39fff3b0f89c:41543 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/39fff3b0f89c:0, corePoolSize=3, maxPoolSize=3 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,486 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,41543,1731956125403-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T18:55:25,497 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T18:55:25,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44515, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T18:55:25,501 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T18:55:25,501 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T18:55:25,501 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T18:55:25,501 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=39fff3b0f89c,41543,1731956125403-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,501 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,501 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.Replication(171): 39fff3b0f89c,41543,1731956125403 started 2024-11-18T18:55:25,501 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T18:55:25,503 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C46121%2C1731956122035.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,46121,1731956122035, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:25,515 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T18:55:25,515 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1482): Serving as 39fff3b0f89c,41543,1731956125403, RpcServer on 39fff3b0f89c/172.17.0.2:41543, sessionid=0x1015096d2060003 2024-11-18T18:55:25,515 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T18:55:25,515 DEBUG [RS:2;39fff3b0f89c:41543 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,515 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,41543,1731956125403' 2024-11-18T18:55:25,515 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T18:55:25,516 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T18:55:25,516 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '39fff3b0f89c,41543,1731956125403' 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T18:55:25,517 DEBUG [RS:2;39fff3b0f89c:41543 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T18:55:25,518 INFO [RS:2;39fff3b0f89c:41543 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T18:55:25,518 INFO [RS:2;39fff3b0f89c:41543 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T18:55:25,523 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,46121,1731956122035/39fff3b0f89c%2C46121%2C1731956122035.meta.1731956125503.meta 2024-11-18T18:55:25,524 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:25,524 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:25,524 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T18:55:25,524 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T18:55:25,524 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T18:55:25,524 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T18:55:25,525 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:25,525 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T18:55:25,525 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T18:55:25,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T18:55:25,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T18:55:25,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:25,528 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438, delay=997 ms 2024-11-18T18:55:25,529 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:25,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:25,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T18:55:25,530 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T18:55:25,530 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:25,532 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508, delay=976 ms 2024-11-18T18:55:25,533 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:25,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:25,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T18:55:25,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T18:55:25,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:25,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:25,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T18:55:25,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T18:55:25,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:25,536 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc, delay=946 ms 2024-11-18T18:55:25,537 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:25,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T18:55:25,538 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T18:55:25,538 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:25,538 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740 2024-11-18T18:55:25,539 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T18:55:25,539 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T18:55:25,539 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T18:55:25,540 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T18:55:25,540 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71083041, jitterRate=0.0592198520898819}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T18:55:25,540 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T18:55:25,541 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731956125525Writing region info on filesystem at 1731956125525Initializing all the Stores at 1731956125525Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956125525Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956125525Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956125525Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731956125525Cleaning up temporary data from old regions at 1731956125539 (+14 ms)Running coprocessor post-open hooks at 1731956125540 (+1 ms)Region opened successfully at 1731956125541 (+1 ms) 2024-11-18T18:55:25,541 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731956125497 2024-11-18T18:55:25,544 DEBUG [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T18:55:25,544 INFO [RS_OPEN_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T18:55:25,544 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:25,545 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 39fff3b0f89c,46121,1731956122035, state=OPEN 2024-11-18T18:55:25,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:25,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:25,550 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T18:55:25,550 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:25,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:25,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:25,551 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T18:55:25,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T18:55:25,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=39fff3b0f89c,46121,1731956122035 in 208 msec 2024-11-18T18:55:25,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T18:55:25,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 370 msec 2024-11-18T18:55:25,555 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 39fff3b0f89c,45879,1731956121790 had 2 regions 2024-11-18T18:55:25,556 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 39fff3b0f89c,45879,1731956121790, splitWal=true, meta=true, isMeta: false 2024-11-18T18:55:25,556 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790-splitting dir is empty, no logs to split. 2024-11-18T18:55:25,556 INFO [PEWorker-3 {}] master.SplitWALManager(105): 39fff3b0f89c,45879,1731956121790 WAL count=0, meta=false 2024-11-18T18:55:25,557 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,45879,1731956121790-splitting dir is empty, no logs to split. 2024-11-18T18:55:25,557 INFO [PEWorker-3 {}] master.SplitWALManager(105): 39fff3b0f89c,45879,1731956121790 WAL count=0, meta=false 2024-11-18T18:55:25,557 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 39fff3b0f89c,45879,1731956121790 WAL splitting is done? wals=0, meta=false 2024-11-18T18:55:25,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN}] 2024-11-18T18:55:25,560 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN 2024-11-18T18:55:25,560 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T18:55:25,618 INFO [RS:2;39fff3b0f89c:41543 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T18:55:25,619 INFO [RS:2;39fff3b0f89c:41543 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=39fff3b0f89c%2C41543%2C1731956125403, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,41543,1731956125403, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs, maxLogs=55 2024-11-18T18:55:25,641 INFO [RS:2;39fff3b0f89c:41543 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/WALs/39fff3b0f89c,41543,1731956125403/39fff3b0f89c%2C41543%2C1731956125403.1731956125620 2024-11-18T18:55:25,642 DEBUG [RS:2;39fff3b0f89c:41543 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T18:55:25,711 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(204): Hosts are {39fff3b0f89c=0} racks are {/default-rack=0} 2024-11-18T18:55:25,712 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T18:55:25,712 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T18:55:25,712 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T18:55:25,712 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T18:55:25,712 INFO [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T18:55:25,713 INFO [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T18:55:25,713 DEBUG [39fff3b0f89c:38159 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T18:55:25,713 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=db9393bb5bef5ac659bddcd971022f24, regionState=OPENING, regionLocation=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:25,716 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 39fff3b0f89c:45879 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:45879 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T18:55:25,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1, error=java.net.ConnectException: Call to address=39fff3b0f89c:45879 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:45879 2024-11-18T18:55:25,721 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-18T18:55:25,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,45879,1731956121790, seqNum=-1 from cache 2024-11-18T18:55:25,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 39fff3b0f89c:45879 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 39fff3b0f89c/172.17.0.2:45879 2024-11-18T18:55:25,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T18:55:25,827 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=39fff3b0f89c,46121,1731956122035, seqNum=-1] 2024-11-18T18:55:25,828 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T18:55:25,831 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48751, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T18:55:25,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN because future has completed 2024-11-18T18:55:25,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,46121,1731956122035}] 2024-11-18T18:55:25,953 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-18T18:55:25,953 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-18T18:55:25,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:55:25,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-18T18:55:25,955 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-18T18:55:25,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-18T18:55:25,991 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:25,992 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => db9393bb5bef5ac659bddcd971022f24, NAME => 'testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.', STARTKEY => '', ENDKEY => ''} 2024-11-18T18:55:25,992 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731956123190 db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:25,992 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T18:55:25,993 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:25,993 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:25,994 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:25,994 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region db9393bb5bef5ac659bddcd971022f24 columnFamilyName family 2024-11-18T18:55:25,994 DEBUG [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T18:55:25,996 DEBUG [StoreFileOpener-db9393bb5bef5ac659bddcd971022f24-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec, delay=923 ms 2024-11-18T18:55:25,998 DEBUG [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec 2024-11-18T18:55:25,998 INFO [StoreOpener-db9393bb5bef5ac659bddcd971022f24-1 {}] regionserver.HStore(327): Store=db9393bb5bef5ac659bddcd971022f24/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T18:55:25,999 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:25,999 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,000 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,000 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,000 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,002 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,002 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened db9393bb5bef5ac659bddcd971022f24; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73102278, jitterRate=0.08930882811546326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T18:55:26,003 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,004 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for db9393bb5bef5ac659bddcd971022f24: Running coprocessor pre-open hook at 1731956125993Writing region info on filesystem at 1731956125993Initializing all the Stores at 1731956125993Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731956125993Cleaning up temporary data from old regions at 1731956126000 (+7 ms)Running coprocessor post-open hooks at 1731956126003 (+3 ms)Region opened successfully at 1731956126004 (+1 ms) 2024-11-18T18:55:26,004 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24., pid=11, masterSystemTime=1731956125988 2024-11-18T18:55:26,007 DEBUG [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,007 INFO [RS_OPEN_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,008 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=db9393bb5bef5ac659bddcd971022f24, regionState=OPEN, openSeqNum=10, regionLocation=39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:26,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,46121,1731956122035 because future has completed 2024-11-18T18:55:26,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-18T18:55:26,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure db9393bb5bef5ac659bddcd971022f24, server=39fff3b0f89c,46121,1731956122035 in 176 msec 2024-11-18T18:55:26,015 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T18:55:26,015 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:55:26,015 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:26,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-18T18:55:26,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,016 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T18:55:26,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731956123190, region=db9393bb5bef5ac659bddcd971022f24, ASSIGN in 455 msec 2024-11-18T18:55:26,016 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 39fff3b0f89c,45879,1731956121790 after splitting done 2024-11-18T18:55:26,016 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T18:55:26,016 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1549915307, stopped=false 2024-11-18T18:55:26,016 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=39fff3b0f89c,38159,1731956121646 2024-11-18T18:55:26,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 39fff3b0f89c,45879,1731956121790, splitWal=true, meta=true in 848 msec 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:26,051 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:55:26,052 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T18:55:26,052 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:26,052 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:26,053 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,46121,1731956122035' ***** 2024-11-18T18:55:26,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:26,053 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:55:26,053 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '39fff3b0f89c,41543,1731956125403' ***** 2024-11-18T18:55:26,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T18:55:26,053 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T18:55:26,053 INFO [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T18:55:26,054 INFO [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T18:55:26,054 INFO [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence 2024-11-18T18:55:26,054 INFO [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/4b7b7cc1-23a0-bc2a-71c9-d3e273aebc1d/bucket.persistence 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T18:55:26,054 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T18:55:26,055 INFO [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T18:55:26,055 INFO [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T18:55:26,055 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T18:55:26,055 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T18:55:26,065 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1487): Thread RS:1;39fff3b0f89c:46121 started persisting bucket cache to file 2024-11-18T18:55:26,065 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1487): Thread RS:2;39fff3b0f89c:41543 started persisting bucket cache to file 2024-11-18T18:55:26,066 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:26,066 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T18:55:26,066 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:26,066 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T18:55:26,068 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:26,068 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T18:55:26,068 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956126065 2024-11-18T18:55:26,068 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731956126065 2024-11-18T18:55:26,068 DEBUG [RS:1;39fff3b0f89c:46121 {}] bucket.BucketCache(1509): Thread RS:1;39fff3b0f89c:46121 finished persisting bucket cache to file, renaming 2024-11-18T18:55:26,068 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:55:26,069 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(3091): Received CLOSE for db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;39fff3b0f89c:46121. 2024-11-18T18:55:26,069 DEBUG [RS:1;39fff3b0f89c:46121 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:26,069 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:26,069 DEBUG [RS:1;39fff3b0f89c:46121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,069 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing db9393bb5bef5ac659bddcd971022f24, disabling compactions & flushes 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:55:26,069 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:55:26,069 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:55:26,069 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. after waiting 0 ms 2024-11-18T18:55:26,069 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T18:55:26,069 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,070 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T18:55:26,070 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1325): Online Regions={db9393bb5bef5ac659bddcd971022f24=testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24., 1588230740=hbase:meta,,1.1588230740} 2024-11-18T18:55:26,070 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T18:55:26,070 DEBUG [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, db9393bb5bef5ac659bddcd971022f24 2024-11-18T18:55:26,070 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T18:55:26,070 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T18:55:26,070 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T18:55:26,070 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T18:55:26,070 DEBUG [StoreFileCloser-db9393bb5bef5ac659bddcd971022f24-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/family/593875e25e044aff94057864221682ec 2024-11-18T18:55:26,070 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T18:55:26,070 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-18T18:55:26,072 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:26,075 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/default/testBlockEvictionOnGracefulStop1731956123190/db9393bb5bef5ac659bddcd971022f24/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T18:55:26,075 INFO [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,075 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for db9393bb5bef5ac659bddcd971022f24: Waiting for close lock at 1731956126069Running coprocessor pre-close hooks at 1731956126069Disabling compacts and flushes for region at 1731956126069Disabling writes for close at 1731956126069Writing region close event to WAL at 1731956126070 (+1 ms)Running coprocessor post-close hooks at 1731956126075 (+5 ms)Closed at 1731956126075 2024-11-18T18:55:26,075 DEBUG [RS_CLOSE_REGION-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24. 2024-11-18T18:55:26,080 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d0bac82f2af24c968ce8c1440cde203b 2024-11-18T18:55:26,086 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T18:55:26,087 INFO [regionserver/39fff3b0f89c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:26,089 DEBUG [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1509): Thread RS:2;39fff3b0f89c:41543 finished persisting bucket cache to file, renaming 2024-11-18T18:55:26,089 WARN [RS:2;39fff3b0f89c:41543 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(959): stopping server 39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:26,089 INFO [RS:2;39fff3b0f89c:41543 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;39fff3b0f89c:41543. 2024-11-18T18:55:26,090 DEBUG [RS:2;39fff3b0f89c:41543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T18:55:26,090 DEBUG [RS:2;39fff3b0f89c:41543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,090 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,41543,1731956125403; all regions closed. 2024-11-18T18:55:26,091 DEBUG [RS:2;39fff3b0f89c:41543 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:26,091 INFO [RS:2;39fff3b0f89c:41543 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C41543%2C1731956125403:(num 1731956125620) 2024-11-18T18:55:26,092 DEBUG [RS:2;39fff3b0f89c:41543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T18:55:26,092 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T18:55:26,092 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:26,093 INFO [RS:2;39fff3b0f89c:41543 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41543 2024-11-18T18:55:26,095 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/d0bac82f2af24c968ce8c1440cde203b is 209, key is testBlockEvictionOnGracefulStop1731956123190,,1731956123196.db9393bb5bef5ac659bddcd971022f24./info:regioninfo/1731956126007/Put/seqid=0 2024-11-18T18:55:26,096 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/d0bac82f2af24c968ce8c1440cde203b 2024-11-18T18:55:26,098 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/.tmp/info/d0bac82f2af24c968ce8c1440cde203b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/d0bac82f2af24c968ce8c1440cde203b 2024-11-18T18:55:26,099 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/d0bac82f2af24c968ce8c1440cde203b, delay=938 ms 2024-11-18T18:55:26,100 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/d0bac82f2af24c968ce8c1440cde203b, entries=8, sequenceid=19, filesize=6.6 K 2024-11-18T18:55:26,101 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false 2024-11-18T18:55:26,101 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/eab3e71fa4fe48aba9d27df7c4ae7438 2024-11-18T18:55:26,102 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/info/d0bac82f2af24c968ce8c1440cde203b 2024-11-18T18:55:26,102 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/ns/49e35eb3bf16499eb1514b6b74a38508 2024-11-18T18:55:26,103 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/table/ed788b7b90334c88b97d60fb643513cc 2024-11-18T18:55:26,106 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-18T18:55:26,107 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T18:55:26,107 INFO [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:26,107 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731956126070Running coprocessor pre-close hooks at 1731956126070Disabling compacts and flushes for region at 1731956126070Disabling writes for close at 1731956126070Obtaining lock to block concurrent updates at 1731956126070Preparing flush snapshotting stores in 1588230740 at 1731956126070Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731956126071 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731956126071Flushing 1588230740/info: creating writer at 1731956126071Flushing 1588230740/info: appending metadata at 1731956126095 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731956126095Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f721a63: reopening flushed file at 1731956126098 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=19, compaction requested=false at 1731956126101 (+3 ms)Writing region close event to WAL at 1731956126103 (+2 ms)Running coprocessor post-close hooks at 1731956126106 (+3 ms)Closed at 1731956126107 (+1 ms) 2024-11-18T18:55:26,107 DEBUG [RS_CLOSE_META-regionserver/39fff3b0f89c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T18:55:26,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,41543,1731956125403 2024-11-18T18:55:26,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:26,126 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:26,134 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,41543,1731956125403] 2024-11-18T18:55:26,142 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,41543,1731956125403 already deleted, retry=false 2024-11-18T18:55:26,142 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,41543,1731956125403 expired; onlineServers=1 2024-11-18T18:55:26,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,234 INFO [RS:2;39fff3b0f89c:41543 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:26,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41543-0x1015096d2060003, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,235 INFO [RS:2;39fff3b0f89c:41543 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,41543,1731956125403; zookeeper connection closed. 2024-11-18T18:55:26,235 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7307d402 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7307d402 2024-11-18T18:55:26,270 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(976): stopping server 39fff3b0f89c,46121,1731956122035; all regions closed. 2024-11-18T18:55:26,276 DEBUG [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:26,276 INFO [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C46121%2C1731956122035.meta:.meta(num 1731956125503) 2024-11-18T18:55:26,281 DEBUG [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/oldWALs 2024-11-18T18:55:26,282 INFO [RS:1;39fff3b0f89c:46121 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 39fff3b0f89c%2C46121%2C1731956122035:(num 1731956122717) 2024-11-18T18:55:26,282 DEBUG [RS:1;39fff3b0f89c:46121 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T18:55:26,282 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T18:55:26,282 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:26,282 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.ChoreService(370): Chore service for: regionserver/39fff3b0f89c:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:26,282 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:26,282 INFO [regionserver/39fff3b0f89c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:26,283 INFO [RS:1;39fff3b0f89c:46121 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46121 2024-11-18T18:55:26,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/39fff3b0f89c,46121,1731956122035 2024-11-18T18:55:26,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T18:55:26,292 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:26,293 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [39fff3b0f89c,46121,1731956122035] 2024-11-18T18:55:26,309 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/39fff3b0f89c,46121,1731956122035 already deleted, retry=false 2024-11-18T18:55:26,309 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 39fff3b0f89c,46121,1731956122035 expired; onlineServers=0 2024-11-18T18:55:26,309 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '39fff3b0f89c,38159,1731956121646' ***** 2024-11-18T18:55:26,309 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T18:55:26,309 INFO [M:0;39fff3b0f89c:38159 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T18:55:26,309 INFO [M:0;39fff3b0f89c:38159 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T18:55:26,309 DEBUG [M:0;39fff3b0f89c:38159 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T18:55:26,310 DEBUG [M:0;39fff3b0f89c:38159 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T18:55:26,310 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T18:55:26,310 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956122394 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.small.0-1731956122394,5,FailOnTimeoutGroup] 2024-11-18T18:55:26,310 DEBUG [master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956122393 {}] cleaner.HFileCleaner(306): Exit Thread[master/39fff3b0f89c:0:becomeActiveMaster-HFileCleaner.large.0-1731956122393,5,FailOnTimeoutGroup] 2024-11-18T18:55:26,310 INFO [M:0;39fff3b0f89c:38159 {}] hbase.ChoreService(370): Chore service for: master/39fff3b0f89c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T18:55:26,310 INFO [M:0;39fff3b0f89c:38159 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T18:55:26,310 DEBUG [M:0;39fff3b0f89c:38159 {}] master.HMaster(1795): Stopping service threads 2024-11-18T18:55:26,311 INFO [M:0;39fff3b0f89c:38159 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T18:55:26,311 INFO [M:0;39fff3b0f89c:38159 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T18:55:26,311 INFO [M:0;39fff3b0f89c:38159 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T18:55:26,311 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T18:55:26,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T18:55:26,317 DEBUG [M:0;39fff3b0f89c:38159 {}] zookeeper.ZKUtil(347): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T18:55:26,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T18:55:26,317 WARN [M:0;39fff3b0f89c:38159 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T18:55:26,318 INFO [M:0;39fff3b0f89c:38159 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/.lastflushedseqids 2024-11-18T18:55:26,325 INFO [M:0;39fff3b0f89c:38159 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T18:55:26,325 INFO [M:0;39fff3b0f89c:38159 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T18:55:26,326 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T18:55:26,326 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:26,326 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:26,326 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T18:55:26,326 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:26,326 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-18T18:55:26,334 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09b21d7df37b461fbc4fade439dd64ef 2024-11-18T18:55:26,345 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09b21d7df37b461fbc4fade439dd64ef is 82, key is hbase:meta,,1/info:regioninfo/1731956125544/Put/seqid=0 2024-11-18T18:55:26,346 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09b21d7df37b461fbc4fade439dd64ef 2024-11-18T18:55:26,352 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6e60dc3a0128405bb82eafef453b189b 2024-11-18T18:55:26,365 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6e60dc3a0128405bb82eafef453b189b is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731956123586/Put/seqid=0 2024-11-18T18:55:26,366 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6e60dc3a0128405bb82eafef453b189b 2024-11-18T18:55:26,372 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 211ef0380a534eabb1d0189a173be508 2024-11-18T18:55:26,385 DEBUG [M:0;39fff3b0f89c:38159 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/211ef0380a534eabb1d0189a173be508 is 69, key is 39fff3b0f89c,41543,1731956125403/rs:state/1731956125467/Put/seqid=0 2024-11-18T18:55:26,386 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/211ef0380a534eabb1d0189a173be508 2024-11-18T18:55:26,388 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 211ef0380a534eabb1d0189a173be508 2024-11-18T18:55:26,388 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09b21d7df37b461fbc4fade439dd64ef as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09b21d7df37b461fbc4fade439dd64ef 2024-11-18T18:55:26,389 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09b21d7df37b461fbc4fade439dd64ef, entries=14, sequenceid=102, filesize=5.9 K 2024-11-18T18:55:26,390 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6e60dc3a0128405bb82eafef453b189b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6e60dc3a0128405bb82eafef453b189b 2024-11-18T18:55:26,391 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6e60dc3a0128405bb82eafef453b189b, entries=11, sequenceid=102, filesize=6.5 K 2024-11-18T18:55:26,391 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/211ef0380a534eabb1d0189a173be508 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/211ef0380a534eabb1d0189a173be508 2024-11-18T18:55:26,392 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 211ef0380a534eabb1d0189a173be508 2024-11-18T18:55:26,392 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/46a25bc6-c75d-a21a-ae70-bd36c23d20ea/bdb97694-7584-b4da-cefa-a6e20dc81b26/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/211ef0380a534eabb1d0189a173be508, entries=3, sequenceid=102, filesize=5.3 K 2024-11-18T18:55:26,393 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false 2024-11-18T18:55:26,395 INFO [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T18:55:26,395 DEBUG [M:0;39fff3b0f89c:38159 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731956126326Disabling compacts and flushes for region at 1731956126326Disabling writes for close at 1731956126326Obtaining lock to block concurrent updates at 1731956126326Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731956126326Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731956126327 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731956126327Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731956126327Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731956126345 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731956126345Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731956126348 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731956126364 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731956126364Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731956126367 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731956126385 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731956126385Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32dec5fc: reopening flushed file at 1731956126388 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34e1bb28: reopening flushed file at 1731956126390 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@680b6553: reopening flushed file at 1731956126391 (+1 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=102, compaction requested=false at 1731956126393 (+2 ms)Writing region close event to WAL at 1731956126395 (+2 ms)Closed at 1731956126395 2024-11-18T18:55:26,396 INFO [M:0;39fff3b0f89c:38159 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T18:55:26,396 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T18:55:26,396 INFO [M:0;39fff3b0f89c:38159 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38159 2024-11-18T18:55:26,396 INFO [M:0;39fff3b0f89c:38159 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T18:55:26,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,401 INFO [RS:1;39fff3b0f89c:46121 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:26,401 INFO [RS:1;39fff3b0f89c:46121 {}] regionserver.HRegionServer(1031): Exiting; stopping=39fff3b0f89c,46121,1731956122035; zookeeper connection closed. 2024-11-18T18:55:26,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46121-0x1015096d2060002, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,401 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@612ac04 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@612ac04 2024-11-18T18:55:26,401 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T18:55:26,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,507 INFO [M:0;39fff3b0f89c:38159 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T18:55:26,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38159-0x1015096d2060000, quorum=127.0.0.1:54605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T18:55:26,513 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T18:55:26,515 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T18:55:26,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T18:55:26,521 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=111 (was 103) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5100 (was 5170)