2024-11-10 19:32:51,921 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 19:32:51,940 main DEBUG Took 0.017590 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 19:32:51,941 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 19:32:51,941 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 19:32:51,943 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 19:32:51,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,951 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 19:32:51,965 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,968 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,970 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,970 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,972 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,972 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,973 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,974 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,975 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,975 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 19:32:51,977 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,977 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 19:32:51,979 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 19:32:51,980 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 19:32:51,981 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 19:32:51,982 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 19:32:51,983 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 19:32:51,984 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 19:32:51,991 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 19:32:51,994 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 19:32:51,996 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 19:32:51,996 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 19:32:51,997 main DEBUG createAppenders(={Console}) 2024-11-10 19:32:51,997 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-10 19:32:51,998 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 19:32:51,998 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-10 19:32:51,999 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 19:32:51,999 main DEBUG OutputStream closed 2024-11-10 19:32:51,999 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 19:32:51,999 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 19:32:52,000 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-10 19:32:52,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 19:32:52,072 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 19:32:52,073 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 19:32:52,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 19:32:52,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 19:32:52,075 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 19:32:52,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 19:32:52,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 19:32:52,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 19:32:52,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 19:32:52,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 19:32:52,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 19:32:52,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 19:32:52,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 19:32:52,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 19:32:52,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 19:32:52,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 19:32:52,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 19:32:52,082 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 19:32:52,083 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-10 19:32:52,083 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 19:32:52,084 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-10T19:32:52,097 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-10 19:32:52,100 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 19:32:52,100 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T19:32:52,329 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f 2024-11-10T19:32:52,353 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=115, ProcessCount=11, AvailableMemoryMB=15972 2024-11-10T19:32:52,444 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T19:32:52,546 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:52,561 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/cluster_f9d6afbf-69ee-207b-1df8-5b43de3d11c5, deleteOnExit=true 2024-11-10T19:32:52,701 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/cluster_f9d6afbf-69ee-207b-1df8-5b43de3d11c5/zookeeper_0, clientPort=53586, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/cluster_f9d6afbf-69ee-207b-1df8-5b43de3d11c5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/cluster_f9d6afbf-69ee-207b-1df8-5b43de3d11c5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T19:32:52,713 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53586 2024-11-10T19:32:52,714 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:52,791 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 with version=8 2024-11-10T19:32:52,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase-staging 2024-11-10T19:32:52,878 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T19:32:53,112 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:53,121 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,123 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,134 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:53,134 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:53,280 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T19:32:53,365 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:32:53,404 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T19:32:53,414 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T19:32:53,417 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:53,454 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 39060 (auto-detected) 2024-11-10T19:32:53,456 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T19:32:53,481 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46433 2024-11-10T19:32:53,501 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46433 connecting to ZooKeeper ensemble=127.0.0.1:53586 2024-11-10T19:32:53,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464330x0, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:53,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46433-0x10127862c080000 connected 2024-11-10T19:32:53,710 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,711 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,719 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:53,722 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5, hbase.cluster.distributed=false 2024-11-10T19:32:53,744 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:53,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-10T19:32:53,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46433 2024-11-10T19:32:53,749 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46433 2024-11-10T19:32:53,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-10T19:32:53,750 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46433 2024-11-10T19:32:53,832 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:53,833 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,834 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:53,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,834 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:53,837 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:32:53,839 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:53,840 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43445 2024-11-10T19:32:53,842 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43445 connecting to ZooKeeper ensemble=127.0.0.1:53586 2024-11-10T19:32:53,843 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,844 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434450x0, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:53,866 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43445-0x10127862c080001 connected 2024-11-10T19:32:53,866 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:53,874 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:32:53,885 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.cache 2024-11-10T19:32:53,890 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T19:32:53,891 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T19:32:53,893 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T19:32:53,894 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T19:32:53,895 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T19:32:53,899 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T19:32:53,903 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T19:32:53,903 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence is complete. 2024-11-10T19:32:53,904 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:32:53,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T19:32:53,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:53,913 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43445 2024-11-10T19:32:53,914 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43445 2024-11-10T19:32:53,915 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43445 2024-11-10T19:32:53,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43445 2024-11-10T19:32:53,916 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43445 2024-11-10T19:32:53,934 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:53,934 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,935 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:53,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:53,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:53,935 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:32:53,936 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:53,937 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43739 2024-11-10T19:32:53,939 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43739 connecting to ZooKeeper ensemble=127.0.0.1:53586 2024-11-10T19:32:53,939 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,940 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:53,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437390x0, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:53,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:437390x0, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:53,966 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43739-0x10127862c080002 connected 2024-11-10T19:32:53,967 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:32:53,968 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.cache 2024-11-10T19:32:53,968 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T19:32:53,969 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T19:32:53,969 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T19:32:53,970 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T19:32:53,970 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T19:32:53,970 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T19:32:53,971 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:32:53,971 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T19:32:53,971 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence is complete. 2024-11-10T19:32:53,972 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T19:32:53,975 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:53,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43739 2024-11-10T19:32:53,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43739 2024-11-10T19:32:53,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43739 2024-11-10T19:32:53,982 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43739 2024-11-10T19:32:53,983 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43739 2024-11-10T19:32:54,002 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8124d42bb870:46433 2024-11-10T19:32:54,003 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,023 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T19:32:54,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T19:32:54,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,066 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T19:32:54,068 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8124d42bb870,46433,1731267172954 from backup master directory 2024-11-10T19:32:54,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:54,079 WARN [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:54,079 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,082 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T19:32:54,084 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T19:32:54,094 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase.id] with ID: a540c228-e8e2-4585-bcfa-82f23c8d5cbb 2024-11-10T19:32:54,094 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.tmp/hbase.id 2024-11-10T19:32:54,100 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase.id] 2024-11-10T19:32:54,133 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:54,139 INFO [master/8124d42bb870:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T19:32:54,160 INFO [master/8124d42bb870:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T19:32:54,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,213 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:32:54,217 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T19:32:54,260 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T19:32:54,267 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:54,319 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store 2024-11-10T19:32:54,339 INFO [master/8124d42bb870:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T19:32:54,342 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:54,344 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:32:54,344 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:54,344 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:54,346 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:32:54,346 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:54,346 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:54,347 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731267174344Disabling compacts and flushes for region at 1731267174344Disabling writes for close at 1731267174346 (+2 ms)Writing region close event to WAL at 1731267174346Closed at 1731267174346 2024-11-10T19:32:54,348 WARN [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/.initializing 2024-11-10T19:32:54,348 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,361 INFO [master/8124d42bb870:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:32:54,375 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8124d42bb870%2C46433%2C1731267172954, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,46433,1731267172954, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/oldWALs, maxLogs=10 2024-11-10T19:32:54,416 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,46433,1731267172954/8124d42bb870%2C46433%2C1731267172954.1731267174380 2024-11-10T19:32:54,417 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:32:54,418 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:54,419 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:54,423 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,424 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,463 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T19:32:54,495 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:54,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:54,498 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T19:32:54,503 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:54,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:54,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,509 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T19:32:54,509 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:54,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:54,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,515 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T19:32:54,515 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:54,516 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:54,516 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,521 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,521 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,525 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,526 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,529 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:32:54,532 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:54,539 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:32:54,540 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64261975, jitterRate=-0.04242195188999176}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:32:54,551 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731267174441Initializing all the Stores at 1731267174442 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267174442Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267174443 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267174444 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267174444Cleaning up temporary data from old regions at 1731267174526 (+82 ms)Region opened successfully at 1731267174550 (+24 ms) 2024-11-10T19:32:54,552 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T19:32:54,591 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb47360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:54,629 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T19:32:54,641 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T19:32:54,641 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T19:32:54,644 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T19:32:54,645 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T19:32:54,651 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T19:32:54,651 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T19:32:54,678 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T19:32:54,688 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T19:32:54,736 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T19:32:54,739 INFO [master/8124d42bb870:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T19:32:54,741 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T19:32:54,752 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T19:32:54,755 INFO [master/8124d42bb870:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T19:32:54,760 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T19:32:54,769 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T19:32:54,771 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T19:32:54,781 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T19:32:54,797 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T19:32:54,806 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,832 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8124d42bb870,46433,1731267172954, sessionid=0x10127862c080000, setting cluster-up flag (Was=false) 2024-11-10T19:32:54,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,889 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T19:32:54,893 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:54,948 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T19:32:54,951 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8124d42bb870,46433,1731267172954 2024-11-10T19:32:54,958 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T19:32:54,987 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(746): ClusterId : a540c228-e8e2-4585-bcfa-82f23c8d5cbb 2024-11-10T19:32:54,987 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(746): ClusterId : a540c228-e8e2-4585-bcfa-82f23c8d5cbb 2024-11-10T19:32:54,989 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:32:54,989 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:32:54,999 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:32:54,999 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:32:55,000 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:32:55,000 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:32:55,012 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:32:55,012 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:32:55,013 DEBUG [RS:0;8124d42bb870:43445 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27c8fac3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:55,013 DEBUG [RS:1;8124d42bb870:43739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2afe31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:55,027 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8124d42bb870:43445 2024-11-10T19:32:55,029 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8124d42bb870:43739 2024-11-10T19:32:55,031 INFO [RS:1;8124d42bb870:43739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:32:55,031 INFO [RS:0;8124d42bb870:43445 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:32:55,032 INFO [RS:1;8124d42bb870:43739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:32:55,032 INFO [RS:0;8124d42bb870:43445 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:32:55,032 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:32:55,032 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:32:55,034 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(2659): reportForDuty to master=8124d42bb870,46433,1731267172954 with port=43739, startcode=1731267173934 2024-11-10T19:32:55,034 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(2659): reportForDuty to master=8124d42bb870,46433,1731267172954 with port=43445, startcode=1731267173799 2024-11-10T19:32:55,044 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:55,045 DEBUG [RS:0;8124d42bb870:43445 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:32:55,045 DEBUG [RS:1;8124d42bb870:43739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:32:55,057 INFO [master/8124d42bb870:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T19:32:55,065 INFO [master/8124d42bb870:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T19:32:55,070 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8124d42bb870,46433,1731267172954 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T19:32:55,077 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:55,077 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:55,077 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:55,077 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:55,078 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8124d42bb870:0, corePoolSize=10, maxPoolSize=10 2024-11-10T19:32:55,078 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,078 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:55,079 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,083 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:55,084 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T19:32:55,084 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731267205084 2024-11-10T19:32:55,086 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T19:32:55,088 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T19:32:55,089 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,090 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T19:32:55,092 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T19:32:55,092 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T19:32:55,092 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T19:32:55,093 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T19:32:55,094 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,098 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T19:32:55,099 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T19:32:55,100 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T19:32:55,104 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T19:32:55,104 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T19:32:55,111 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267175105,5,FailOnTimeoutGroup] 2024-11-10T19:32:55,111 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267175111,5,FailOnTimeoutGroup] 2024-11-10T19:32:55,112 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,112 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T19:32:55,113 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33629, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:32:55,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41003, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:32:55,113 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,113 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,118 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T19:32:55,119 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:55,120 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,124 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46433 {}] master.ServerManager(517): Registering regionserver=8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,125 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:55,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T19:32:55,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:32:55,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T19:32:55,136 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:32:55,136 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46433 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,136 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,136 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46433 {}] master.ServerManager(517): Registering regionserver=8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,136 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T19:32:55,140 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:55,140 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T19:32:55,140 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:32:55,143 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:55,143 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:32:55,143 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T19:32:55,144 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:32:55,144 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T19:32:55,148 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:32:55,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,149 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:32:55,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:55,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:55,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:32:55,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:32:55,151 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:32:55,151 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:32:55,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:32:55,159 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:32:55,160 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63811844, jitterRate=-0.0491294264793396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:32:55,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731267175126Initializing all the Stores at 1731267175126Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175126Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175126Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267175126Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175126Cleaning up temporary data from old regions at 1731267175150 (+24 ms)Region opened successfully at 1731267175166 (+16 ms) 2024-11-10T19:32:55,166 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:32:55,166 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:32:55,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:32:55,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:32:55,167 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:32:55,168 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:32:55,169 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731267175166Disabling compacts and flushes for region at 1731267175166Disabling writes for close at 1731267175167 (+1 ms)Writing region close event to WAL at 1731267175168 (+1 ms)Closed at 1731267175168 2024-11-10T19:32:55,171 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:55,171 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T19:32:55,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T19:32:55,187 DEBUG [RS:1;8124d42bb870:43739 {}] zookeeper.ZKUtil(111): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,187 WARN [RS:1;8124d42bb870:43739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:55,188 DEBUG [RS:0;8124d42bb870:43445 {}] zookeeper.ZKUtil(111): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,188 INFO [RS:1;8124d42bb870:43739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:55,188 WARN [RS:0;8124d42bb870:43445 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:55,188 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,188 INFO [RS:0;8124d42bb870:43445 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:55,188 DEBUG [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8124d42bb870,43445,1731267173799] 2024-11-10T19:32:55,189 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8124d42bb870,43739,1731267173934] 2024-11-10T19:32:55,191 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:32:55,194 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T19:32:55,216 INFO [RS:1;8124d42bb870:43739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:32:55,216 INFO [RS:0;8124d42bb870:43445 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:32:55,234 INFO [RS:1;8124d42bb870:43739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:32:55,234 INFO [RS:0;8124d42bb870:43445 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:32:55,239 INFO [RS:1;8124d42bb870:43739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:32:55,239 INFO [RS:0;8124d42bb870:43445 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:32:55,239 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,239 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,240 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:32:55,240 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:32:55,245 INFO [RS:1;8124d42bb870:43739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:32:55,245 INFO [RS:0;8124d42bb870:43445 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:32:55,247 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,247 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,247 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,247 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,247 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,247 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,247 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,247 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,248 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,248 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,248 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,248 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,248 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:55,248 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:55,248 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,249 DEBUG [RS:1;8124d42bb870:43739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:55,249 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:55,250 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:55,250 DEBUG [RS:0;8124d42bb870:43445 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:55,254 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,254 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,43445,1731267173799-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:32:55,255 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,43739,1731267173934-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:32:55,287 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:32:55,287 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:32:55,289 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,43445,1731267173799-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,289 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,43739,1731267173934-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,290 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,290 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,290 INFO [RS:1;8124d42bb870:43739 {}] regionserver.Replication(171): 8124d42bb870,43739,1731267173934 started 2024-11-10T19:32:55,290 INFO [RS:0;8124d42bb870:43445 {}] regionserver.Replication(171): 8124d42bb870,43445,1731267173799 started 2024-11-10T19:32:55,313 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,313 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1482): Serving as 8124d42bb870,43739,1731267173934, RpcServer on 8124d42bb870/172.17.0.2:43739, sessionid=0x10127862c080002 2024-11-10T19:32:55,315 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:32:55,315 DEBUG [RS:1;8124d42bb870:43739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,315 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,43739,1731267173934' 2024-11-10T19:32:55,315 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:55,315 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:32:55,315 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1482): Serving as 8124d42bb870,43445,1731267173799, RpcServer on 8124d42bb870/172.17.0.2:43445, sessionid=0x10127862c080001 2024-11-10T19:32:55,316 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:32:55,316 DEBUG [RS:0;8124d42bb870:43445 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,316 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,43445,1731267173799' 2024-11-10T19:32:55,316 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:32:55,317 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:32:55,317 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:32:55,317 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:32:55,317 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:32:55,318 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:32:55,318 DEBUG [RS:1;8124d42bb870:43739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,318 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:32:55,318 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,43739,1731267173934' 2024-11-10T19:32:55,318 DEBUG [RS:0;8124d42bb870:43445 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8124d42bb870,43445,1731267173799 2024-11-10T19:32:55,318 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:32:55,318 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,43445,1731267173799' 2024-11-10T19:32:55,318 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:32:55,318 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:32:55,319 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:32:55,319 DEBUG [RS:1;8124d42bb870:43739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:32:55,319 DEBUG [RS:0;8124d42bb870:43445 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:32:55,319 INFO [RS:1;8124d42bb870:43739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:32:55,319 INFO [RS:0;8124d42bb870:43445 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:32:55,319 INFO [RS:1;8124d42bb870:43739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:32:55,319 INFO [RS:0;8124d42bb870:43445 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:32:55,346 WARN [8124d42bb870:46433 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T19:32:55,425 INFO [RS:1;8124d42bb870:43739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:32:55,425 INFO [RS:0;8124d42bb870:43445 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:32:55,426 INFO [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C43739%2C1731267173934, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43739,1731267173934, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:32:55,426 INFO [RS:0;8124d42bb870:43445 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C43445%2C1731267173799, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43445,1731267173799, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:32:55,447 INFO [RS:0;8124d42bb870:43445 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43445,1731267173799/8124d42bb870%2C43445%2C1731267173799.1731267175431 2024-11-10T19:32:55,448 DEBUG [RS:0;8124d42bb870:43445 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:32:55,450 INFO [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43739,1731267173934/8124d42bb870%2C43739%2C1731267173934.1731267175431 2024-11-10T19:32:55,451 DEBUG [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:32:55,599 DEBUG [8124d42bb870:46433 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T19:32:55,612 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(204): Hosts are {8124d42bb870=0} racks are {/default-rack=0} 2024-11-10T19:32:55,615 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T19:32:55,615 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T19:32:55,615 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T19:32:55,615 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T19:32:55,615 INFO [8124d42bb870:46433 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T19:32:55,615 INFO [8124d42bb870:46433 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T19:32:55,615 DEBUG [8124d42bb870:46433 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T19:32:55,621 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,626 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8124d42bb870,43739,1731267173934, state=OPENING 2024-11-10T19:32:55,677 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T19:32:55,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:55,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:55,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:55,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,691 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,694 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:32:55,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8124d42bb870,43739,1731267173934}] 2024-11-10T19:32:55,875 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T19:32:55,879 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T19:32:55,887 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T19:32:55,887 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:55,888 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T19:32:55,889 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C43739%2C1731267173934.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43739,1731267173934, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:32:55,907 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,43739,1731267173934/8124d42bb870%2C43739%2C1731267173934.meta.1731267175891.meta 2024-11-10T19:32:55,907 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:32:55,907 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:55,909 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T19:32:55,911 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T19:32:55,915 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T19:32:55,918 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T19:32:55,918 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:55,919 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T19:32:55,919 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T19:32:55,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T19:32:55,921 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:32:55,921 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T19:32:55,922 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:32:55,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T19:32:55,924 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:32:55,924 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T19:32:55,925 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:32:55,925 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:55,925 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:55,925 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:32:55,926 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:55,927 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:55,927 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:32:55,927 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:32:55,928 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:32:55,929 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:32:55,929 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63692283, jitterRate=-0.05091102421283722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:32:55,929 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T19:32:55,931 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731267175919Writing region info on filesystem at 1731267175919Initializing all the Stores at 1731267175920 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175920Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175920Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267175920Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267175920Cleaning up temporary data from old regions at 1731267175927 (+7 ms)Running coprocessor post-open hooks at 1731267175929 (+2 ms)Region opened successfully at 1731267175931 (+2 ms) 2024-11-10T19:32:55,936 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731267175867 2024-11-10T19:32:55,945 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T19:32:55,945 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T19:32:55,946 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,947 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8124d42bb870,43739,1731267173934, state=OPEN 2024-11-10T19:32:55,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:32:55,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:32:55,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:32:55,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,973 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:32:55,973 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8124d42bb870,43739,1731267173934 2024-11-10T19:32:55,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T19:32:55,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8124d42bb870,43739,1731267173934 in 276 msec 2024-11-10T19:32:55,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T19:32:55,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 798 msec 2024-11-10T19:32:55,984 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:55,984 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T19:32:56,005 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:32:56,007 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8124d42bb870,43739,1731267173934, seqNum=-1] 2024-11-10T19:32:56,026 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:32:56,028 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45359, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:32:56,043 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0460 sec 2024-11-10T19:32:56,043 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731267176043, completionTime=-1 2024-11-10T19:32:56,046 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T19:32:56,047 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T19:32:56,073 INFO [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T19:32:56,073 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731267236073 2024-11-10T19:32:56,073 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731267296073 2024-11-10T19:32:56,073 INFO [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-10T19:32:56,075 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-10T19:32:56,081 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,081 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,082 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,083 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8124d42bb870:46433, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,083 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,084 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,090 DEBUG [master/8124d42bb870:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T19:32:56,112 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.031sec 2024-11-10T19:32:56,113 INFO [master/8124d42bb870:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T19:32:56,114 INFO [master/8124d42bb870:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T19:32:56,115 INFO [master/8124d42bb870:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T19:32:56,116 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T19:32:56,116 INFO [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T19:32:56,116 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:32:56,117 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T19:32:56,118 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T19:32:56,119 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T19:32:56,119 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46433,1731267172954-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:56,197 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71985066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:32:56,200 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T19:32:56,200 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T19:32:56,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8124d42bb870,46433,-1 for getting cluster id 2024-11-10T19:32:56,206 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:32:56,213 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a540c228-e8e2-4585-bcfa-82f23c8d5cbb' 2024-11-10T19:32:56,216 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:32:56,216 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a540c228-e8e2-4585-bcfa-82f23c8d5cbb" 2024-11-10T19:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2314353, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:32:56,218 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8124d42bb870,46433,-1] 2024-11-10T19:32:56,221 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:32:56,222 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:56,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:32:56,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e124b8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:32:56,227 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:32:56,234 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8124d42bb870,43739,1731267173934, seqNum=-1] 2024-11-10T19:32:56,235 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:32:56,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:32:56,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8124d42bb870,46433,1731267172954 2024-11-10T19:32:56,275 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:32:56,280 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8124d42bb870,46433,1731267172954 2024-11-10T19:32:56,282 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fbe144e 2024-11-10T19:32:56,283 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:32:56,285 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:32:56,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731267176268', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:32:56,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 2024-11-10T19:32:56,301 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:32:56,303 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731267176268" procId is: 4 2024-11-10T19:32:56,304 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:56,306 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:32:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:32:56,318 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8abe4ee7fcec29a123805789609b5882, NAME => 'testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731267176268', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1722): Closing 8abe4ee7fcec29a123805789609b5882, disabling compactions & flushes 2024-11-10T19:32:56,326 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,326 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. after waiting 0 ms 2024-11-10T19:32:56,327 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,327 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,327 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731267176268-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8abe4ee7fcec29a123805789609b5882: Waiting for close lock at 1731267176326Disabling compacts and flushes for region at 1731267176326Disabling writes for close at 1731267176327 (+1 ms)Writing region close event to WAL at 1731267176327Closed at 1731267176327 2024-11-10T19:32:56,328 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:32:56,333 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731267176328"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731267176328"}]},"ts":"1731267176328"} 2024-11-10T19:32:56,338 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:32:56,340 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:32:56,343 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731267176268","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731267176340"}]},"ts":"1731267176340"} 2024-11-10T19:32:56,346 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731267176268, state=ENABLING in hbase:meta 2024-11-10T19:32:56,347 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {8124d42bb870=0} racks are {/default-rack=0} 2024-11-10T19:32:56,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T19:32:56,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T19:32:56,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T19:32:56,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T19:32:56,348 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T19:32:56,348 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T19:32:56,348 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T19:32:56,350 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, ASSIGN}] 2024-11-10T19:32:56,352 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, ASSIGN 2024-11-10T19:32:56,353 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, ASSIGN; state=OFFLINE, location=8124d42bb870,43445,1731267173799; forceNewPlan=false, retain=false 2024-11-10T19:32:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:32:56,509 INFO [8124d42bb870:46433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T19:32:56,511 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=OPENING, regionLocation=8124d42bb870,43445,1731267173799 2024-11-10T19:32:56,516 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, ASSIGN because future has completed 2024-11-10T19:32:56,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799}] 2024-11-10T19:32:56,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:32:56,671 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T19:32:56,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40645, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T19:32:56,676 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,676 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8abe4ee7fcec29a123805789609b5882, NAME => 'testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:56,677 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731267176268 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,677 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:56,677 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,678 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,679 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,683 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8abe4ee7fcec29a123805789609b5882 columnFamilyName family 2024-11-10T19:32:56,683 DEBUG [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:56,683 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(327): Store=8abe4ee7fcec29a123805789609b5882/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:56,683 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,685 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,685 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,685 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,685 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,686 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,691 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:32:56,692 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8abe4ee7fcec29a123805789609b5882; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63530754, jitterRate=-0.05331799387931824}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T19:32:56,692 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:56,693 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8abe4ee7fcec29a123805789609b5882: Running coprocessor pre-open hook at 1731267176678Writing region info on filesystem at 1731267176678Initializing all the Stores at 1731267176678Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267176678Cleaning up temporary data from old regions at 1731267176685 (+7 ms)Running coprocessor post-open hooks at 1731267176692 (+7 ms)Region opened successfully at 1731267176693 (+1 ms) 2024-11-10T19:32:56,694 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882., pid=6, masterSystemTime=1731267176670 2024-11-10T19:32:56,697 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,697 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=OPEN, openSeqNum=2, regionLocation=8124d42bb870,43445,1731267173799 2024-11-10T19:32:56,697 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:56,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799 because future has completed 2024-11-10T19:32:56,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T19:32:56,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799 in 184 msec 2024-11-10T19:32:56,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T19:32:56,705 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, ASSIGN in 354 msec 2024-11-10T19:32:56,706 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:32:56,706 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731267176268","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731267176706"}]},"ts":"1731267176706"} 2024-11-10T19:32:56,709 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731267176268, state=ENABLED in hbase:meta 2024-11-10T19:32:56,710 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:32:56,712 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731267176268 in 416 msec 2024-11-10T19:32:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:32:56,944 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731267176268 completed 2024-11-10T19:32:56,944 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731267176268 get assigned. Timeout = 60000ms 2024-11-10T19:32:56,948 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T19:32:56,955 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731267176268 assigned to meta. Checking AM states. 2024-11-10T19:32:56,956 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T19:32:56,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731267176268 assigned. 2024-11-10T19:32:56,965 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731267176268', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882., hostname=8124d42bb870,43445,1731267173799, seqNum=2] 2024-11-10T19:32:56,966 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:32:56,968 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:32:56,977 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8abe4ee7fcec29a123805789609b5882 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T19:32:56,999 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:57,033 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/.tmp/family/1e6e40f21ae3435dbd29bc13e60a0215 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T19:32:57,038 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/.tmp/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:57,067 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/.tmp/family/1e6e40f21ae3435dbd29bc13e60a0215 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:57,068 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, delay=935 ms 2024-11-10T19:32:57,070 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T19:32:57,075 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8abe4ee7fcec29a123805789609b5882 in 97ms, sequenceid=6, compaction requested=false 2024-11-10T19:32:57,076 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731267176268' 2024-11-10T19:32:57,078 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8abe4ee7fcec29a123805789609b5882: 2024-11-10T19:32:58,006 DEBUG [hfile-prefetch-1731267177069 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1e6e40f21ae3435dbd29bc13e60a0215_0. 2024-11-10T19:32:58,011 DEBUG [hfile-prefetch-1731267177069 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1e6e40f21ae3435dbd29bc13e60a0215_117. 2024-11-10T19:32:58,012 DEBUG [hfile-prefetch-1731267177069 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, with total blocks 2, and data blocks 1 2024-11-10T19:32:58,012 DEBUG [hfile-prefetch-1731267177069 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T19:32:58,012 DEBUG [hfile-prefetch-1731267177069 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T19:32:58,013 DEBUG [hfile-prefetch-1731267177069 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215. Will sleep 100ms and try the verification again. 2024-11-10T19:32:58,089 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=8abe4ee7fcec29a123805789609b5882, source=8124d42bb870,43445,1731267173799, destination=8124d42bb870,43739,1731267173934, warming up region on 8124d42bb870,43739,1731267173934 2024-11-10T19:32:58,092 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=8abe4ee7fcec29a123805789609b5882, source=8124d42bb870,43445,1731267173799, destination=8124d42bb870,43739,1731267173934, running balancer 2024-11-10T19:32:58,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE 2024-11-10T19:32:58,094 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE 2024-11-10T19:32:58,094 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(7855): Warmup {ENCODED => 8abe4ee7fcec29a123805789609b5882, NAME => 'testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:58,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:58,095 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=CLOSING, regionLocation=8124d42bb870,43445,1731267173799 2024-11-10T19:32:58,096 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8abe4ee7fcec29a123805789609b5882 columnFamilyName family 2024-11-10T19:32:58,096 DEBUG [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:58,097 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T19:32:58,097 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T19:32:58,097 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:32:58,097 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:32:58,100 DEBUG [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:58,100 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(327): Store=8abe4ee7fcec29a123805789609b5882/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:58,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1722): Closing 8abe4ee7fcec29a123805789609b5882, disabling compactions & flushes 2024-11-10T19:32:58,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE because future has completed 2024-11-10T19:32:58,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. after waiting 0 ms 2024-11-10T19:32:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,101 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-10T19:32:58,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799}] 2024-11-10T19:32:58,102 DEBUG [StoreFileCloser-8abe4ee7fcec29a123805789609b5882-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:58,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43739 {}] regionserver.HRegion(1676): Region close journal for 8abe4ee7fcec29a123805789609b5882: Waiting for close lock at 1731267178100Disabling compacts and flushes for region at 1731267178100Disabling writes for close at 1731267178101 (+1 ms)Writing region close event to WAL at 1731267178102 (+1 ms)Closed at 1731267178102 2024-11-10T19:32:58,104 WARN [hfile-prefetch-1731267177069 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T19:32:58,105 DEBUG [hfile-prefetch-1731267177069 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:58,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T19:32:58,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731267178097 2024-11-10T19:32:58,117 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T19:32:58,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46433 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-10T19:32:58,265 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,265 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-10T19:32:58,267 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 8abe4ee7fcec29a123805789609b5882, disabling compactions & flushes 2024-11-10T19:32:58,268 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,268 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,268 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. after waiting 0 ms 2024-11-10T19:32:58,269 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,270 DEBUG [StoreFileCloser-8abe4ee7fcec29a123805789609b5882-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 1e6e40f21ae3435dbd29bc13e60a0215, starting offset: 0, end offset: 9223372036854775807 2024-11-10T19:32:58,271 DEBUG [StoreFileCloser-8abe4ee7fcec29a123805789609b5882-family-1 {}] bucket.BucketCache(857): removed key 1e6e40f21ae3435dbd29bc13e60a0215_0 from back map with offset lock 2094080 in the evict process 2024-11-10T19:32:58,274 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T19:32:58,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T19:32:58,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:32:58,274 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:32:58,277 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T19:32:58,278 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:32:58,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267178274 2024-11-10T19:32:58,278 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 8abe4ee7fcec29a123805789609b5882: Waiting for close lock at 1731267178267Running coprocessor pre-close hooks at 1731267178267Disabling compacts and flushes for region at 1731267178267Disabling writes for close at 1731267178268 (+1 ms)Writing region close event to WAL at 1731267178272 (+4 ms)Running coprocessor post-close hooks at 1731267178277 (+5 ms)Closed at 1731267178278 (+1 ms) 2024-11-10T19:32:58,278 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T19:32:58,278 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 8abe4ee7fcec29a123805789609b5882 move to 8124d42bb870,43739,1731267173934 record at close sequenceid=6 2024-11-10T19:32:58,282 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,282 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=CLOSED 2024-11-10T19:32:58,285 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799 because future has completed 2024-11-10T19:32:58,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T19:32:58,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43445,1731267173799 in 186 msec 2024-11-10T19:32:58,290 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE; state=CLOSED, location=8124d42bb870,43739,1731267173934; forceNewPlan=false, retain=false 2024-11-10T19:32:58,440 INFO [8124d42bb870:46433 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T19:32:58,441 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=OPENING, regionLocation=8124d42bb870,43739,1731267173934 2024-11-10T19:32:58,446 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE because future has completed 2024-11-10T19:32:58,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43739,1731267173934}] 2024-11-10T19:32:58,605 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,606 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 8abe4ee7fcec29a123805789609b5882, NAME => 'testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:58,606 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731267176268 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,607 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:58,607 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,607 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,609 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,610 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8abe4ee7fcec29a123805789609b5882 columnFamilyName family 2024-11-10T19:32:58,610 DEBUG [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:58,613 DEBUG [StoreFileOpener-8abe4ee7fcec29a123805789609b5882-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215, delay=955 ms 2024-11-10T19:32:58,614 DEBUG [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:58,615 INFO [StoreOpener-8abe4ee7fcec29a123805789609b5882-1 {}] regionserver.HStore(327): Store=8abe4ee7fcec29a123805789609b5882/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:58,615 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,616 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,616 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,616 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,617 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,621 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,622 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 8abe4ee7fcec29a123805789609b5882; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73173822, jitterRate=0.0903749167919159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T19:32:58,622 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,623 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 8abe4ee7fcec29a123805789609b5882: Running coprocessor pre-open hook at 1731267178608Writing region info on filesystem at 1731267178608Initializing all the Stores at 1731267178608Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267178608Cleaning up temporary data from old regions at 1731267178617 (+9 ms)Running coprocessor post-open hooks at 1731267178622 (+5 ms)Region opened successfully at 1731267178623 (+1 ms) 2024-11-10T19:32:58,624 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882., pid=9, masterSystemTime=1731267178600 2024-11-10T19:32:58,626 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,626 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,626 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=8abe4ee7fcec29a123805789609b5882, regionState=OPEN, openSeqNum=10, regionLocation=8124d42bb870,43739,1731267173934 2024-11-10T19:32:58,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43739,1731267173934 because future has completed 2024-11-10T19:32:58,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T19:32:58,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 8abe4ee7fcec29a123805789609b5882, server=8124d42bb870,43739,1731267173934 in 185 msec 2024-11-10T19:32:58,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731267176268, region=8abe4ee7fcec29a123805789609b5882, REOPEN/MOVE in 540 msec 2024-11-10T19:32:58,646 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T19:32:58,647 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:32:58,647 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:32:58,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,652 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:32:58,652 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T19:32:58,652 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-10T19:32:58,653 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8124d42bb870,46433,1731267172954 2024-11-10T19:32:58,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:58,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:58,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:58,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:58,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:58,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:58,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:32:58,686 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:32:58,687 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:32:58,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:58,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:58,688 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8124d42bb870,43445,1731267173799' ***** 2024-11-10T19:32:58,688 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:58,688 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T19:32:58,688 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8124d42bb870,43739,1731267173934' ***** 2024-11-10T19:32:58,688 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T19:32:58,688 INFO [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T19:32:58,689 INFO [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T19:32:58,689 INFO [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence 2024-11-10T19:32:58,689 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T19:32:58,689 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T19:32:58,689 INFO [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/bucket.persistence 2024-11-10T19:32:58,689 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T19:32:58,689 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T19:32:58,689 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T19:32:58,690 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T19:32:58,690 INFO [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T19:32:58,690 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T19:32:58,690 INFO [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T19:32:58,690 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T19:32:58,700 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1488): Thread RS:1;8124d42bb870:43739 started persisting bucket cache to file 2024-11-10T19:32:58,700 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:32:58,700 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:32:58,700 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1488): Thread RS:0;8124d42bb870:43445 started persisting bucket cache to file 2024-11-10T19:32:58,700 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:32:58,700 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:32:58,703 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:32:58,703 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:32:58,703 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267178700 2024-11-10T19:32:58,703 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267178700 2024-11-10T19:32:58,703 DEBUG [RS:1;8124d42bb870:43739 {}] bucket.BucketCache(1510): Thread RS:1;8124d42bb870:43739 finished persisting bucket cache to file, renaming 2024-11-10T19:32:58,703 DEBUG [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1510): Thread RS:0;8124d42bb870:43445 finished persisting bucket cache to file, renaming 2024-11-10T19:32:58,703 WARN [RS:0;8124d42bb870:43445 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-10T19:32:58,703 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:32:58,703 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:32:58,704 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:32:58,704 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(959): stopping server 8124d42bb870,43445,1731267173799 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(3091): Received CLOSE for 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8124d42bb870:43445. 2024-11-10T19:32:58,704 DEBUG [RS:0;8124d42bb870:43445 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:32:58,704 DEBUG [RS:0;8124d42bb870:43445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,704 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(976): stopping server 8124d42bb870,43445,1731267173799; all regions closed. 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(959): stopping server 8124d42bb870,43739,1731267173934 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:32:58,704 INFO [RS:1;8124d42bb870:43739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8124d42bb870:43739. 2024-11-10T19:32:58,705 DEBUG [RS:1;8124d42bb870:43739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:32:58,705 DEBUG [RS:1;8124d42bb870:43739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,705 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T19:32:58,705 INFO [RS:1;8124d42bb870:43739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:32:58,705 INFO [RS:1;8124d42bb870:43739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:32:58,705 INFO [RS:1;8124d42bb870:43739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:32:58,705 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T19:32:58,705 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8abe4ee7fcec29a123805789609b5882, disabling compactions & flushes 2024-11-10T19:32:58,705 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,705 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,705 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. after waiting 0 ms 2024-11-10T19:32:58,706 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,706 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T19:32:58,706 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T19:32:58,706 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1325): Online Regions={8abe4ee7fcec29a123805789609b5882=testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T19:32:58,706 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:32:58,706 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:32:58,706 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:32:58,706 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:32:58,706 DEBUG [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8abe4ee7fcec29a123805789609b5882 2024-11-10T19:32:58,706 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:32:58,706 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-10T19:32:58,707 DEBUG [StoreFileCloser-8abe4ee7fcec29a123805789609b5882-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/family/1e6e40f21ae3435dbd29bc13e60a0215 2024-11-10T19:32:58,708 DEBUG [RS:0;8124d42bb870:43445 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C43445%2C1731267173799:(num 1731267175431) 2024-11-10T19:32:58,709 DEBUG [RS:0;8124d42bb870:43445 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] hbase.ChoreService(370): Chore service for: regionserver/8124d42bb870:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:32:58,709 INFO [regionserver/8124d42bb870:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:32:58,709 INFO [RS:0;8124d42bb870:43445 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:32:58,710 INFO [RS:0;8124d42bb870:43445 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:32:58,710 INFO [RS:0;8124d42bb870:43445 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:32:58,710 INFO [RS:0;8124d42bb870:43445 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43445 2024-11-10T19:32:58,711 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnRegionMove1731267176268/8abe4ee7fcec29a123805789609b5882/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T19:32:58,712 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,712 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8abe4ee7fcec29a123805789609b5882: Waiting for close lock at 1731267178705Running coprocessor pre-close hooks at 1731267178705Disabling compacts and flushes for region at 1731267178705Disabling writes for close at 1731267178705Writing region close event to WAL at 1731267178707 (+2 ms)Running coprocessor post-close hooks at 1731267178712 (+5 ms)Closed at 1731267178712 2024-11-10T19:32:58,712 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882. 2024-11-10T19:32:58,716 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 536bc7a413d847fa94d13df0c95377c2 2024-11-10T19:32:58,731 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/536bc7a413d847fa94d13df0c95377c2 is 205, key is testBlockEvictionOnRegionMove1731267176268,,1731267176287.8abe4ee7fcec29a123805789609b5882./info:regioninfo/1731267178626/Put/seqid=0 2024-11-10T19:32:58,732 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/536bc7a413d847fa94d13df0c95377c2 2024-11-10T19:32:58,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8124d42bb870,43445,1731267173799 2024-11-10T19:32:58,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:32:58,735 INFO [RS:0;8124d42bb870:43445 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:32:58,736 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8124d42bb870,43445,1731267173799] 2024-11-10T19:32:58,740 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3528035806ff44ce83f41ab8b01b08a2 2024-11-10T19:32:58,753 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/3528035806ff44ce83f41ab8b01b08a2 is 43, key is default/ns:d/1731267176032/Put/seqid=0 2024-11-10T19:32:58,754 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/3528035806ff44ce83f41ab8b01b08a2 2024-11-10T19:32:58,755 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8124d42bb870,43445,1731267173799 already deleted, retry=false 2024-11-10T19:32:58,755 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8124d42bb870,43445,1731267173799 expired; onlineServers=1 2024-11-10T19:32:58,761 INFO [regionserver/8124d42bb870:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:32:58,761 INFO [regionserver/8124d42bb870:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:32:58,762 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 380ac87e93624ec4a69780bf0cc178ce 2024-11-10T19:32:58,773 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/380ac87e93624ec4a69780bf0cc178ce is 78, key is testBlockEvictionOnRegionMove1731267176268/table:state/1731267176706/Put/seqid=0 2024-11-10T19:32:58,774 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/380ac87e93624ec4a69780bf0cc178ce 2024-11-10T19:32:58,777 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/536bc7a413d847fa94d13df0c95377c2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/536bc7a413d847fa94d13df0c95377c2 2024-11-10T19:32:58,778 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/536bc7a413d847fa94d13df0c95377c2, delay=933 ms 2024-11-10T19:32:58,779 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/536bc7a413d847fa94d13df0c95377c2, entries=15, sequenceid=15, filesize=7.6 K 2024-11-10T19:32:58,779 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/3528035806ff44ce83f41ab8b01b08a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/3528035806ff44ce83f41ab8b01b08a2 2024-11-10T19:32:58,780 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/3528035806ff44ce83f41ab8b01b08a2, delay=946 ms 2024-11-10T19:32:58,782 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/3528035806ff44ce83f41ab8b01b08a2, entries=2, sequenceid=15, filesize=5.0 K 2024-11-10T19:32:58,782 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/380ac87e93624ec4a69780bf0cc178ce as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/380ac87e93624ec4a69780bf0cc178ce 2024-11-10T19:32:58,783 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/380ac87e93624ec4a69780bf0cc178ce, delay=999 ms 2024-11-10T19:32:58,784 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/380ac87e93624ec4a69780bf0cc178ce, entries=2, sequenceid=15, filesize=5.3 K 2024-11-10T19:32:58,785 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false 2024-11-10T19:32:58,785 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T19:32:58,787 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/536bc7a413d847fa94d13df0c95377c2 2024-11-10T19:32:58,787 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/3528035806ff44ce83f41ab8b01b08a2 2024-11-10T19:32:58,788 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/380ac87e93624ec4a69780bf0cc178ce 2024-11-10T19:32:58,792 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-10T19:32:58,793 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T19:32:58,793 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:32:58,793 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731267178706Running coprocessor pre-close hooks at 1731267178706Disabling compacts and flushes for region at 1731267178706Disabling writes for close at 1731267178706Obtaining lock to block concurrent updates at 1731267178706Preparing flush snapshotting stores in 1588230740 at 1731267178706Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731267178707 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731267178708 (+1 ms)Flushing 1588230740/info: creating writer at 1731267178708Flushing 1588230740/info: appending metadata at 1731267178730 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731267178730Flushing 1588230740/ns: creating writer at 1731267178734 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731267178752 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731267178752Flushing 1588230740/table: creating writer at 1731267178756 (+4 ms)Flushing 1588230740/table: appending metadata at 1731267178773 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731267178773Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4466d3bc: reopening flushed file at 1731267178776 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eecedd0: reopening flushed file at 1731267178779 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30972533: reopening flushed file at 1731267178782 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 79ms, sequenceid=15, compaction requested=false at 1731267178785 (+3 ms)Writing region close event to WAL at 1731267178788 (+3 ms)Running coprocessor post-close hooks at 1731267178793 (+5 ms)Closed at 1731267178793 2024-11-10T19:32:58,793 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T19:32:58,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:58,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43445-0x10127862c080001, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:58,848 INFO [RS:0;8124d42bb870:43445 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:32:58,849 INFO [RS:0;8124d42bb870:43445 {}] regionserver.HRegionServer(1031): Exiting; stopping=8124d42bb870,43445,1731267173799; zookeeper connection closed. 2024-11-10T19:32:58,849 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5dc9b187 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5dc9b187 2024-11-10T19:32:58,906 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(976): stopping server 8124d42bb870,43739,1731267173934; all regions closed. 2024-11-10T19:32:58,913 DEBUG [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:32:58,913 INFO [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C43739%2C1731267173934.meta:.meta(num 1731267175891) 2024-11-10T19:32:58,916 DEBUG [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:32:58,916 INFO [RS:1;8124d42bb870:43739 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C43739%2C1731267173934:(num 1731267175431) 2024-11-10T19:32:58,916 DEBUG [RS:1;8124d42bb870:43739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:32:58,916 INFO [RS:1;8124d42bb870:43739 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:32:58,916 INFO [RS:1;8124d42bb870:43739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:32:58,916 INFO [RS:1;8124d42bb870:43739 {}] hbase.ChoreService(370): Chore service for: regionserver/8124d42bb870:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T19:32:58,916 INFO [RS:1;8124d42bb870:43739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:32:58,917 INFO [regionserver/8124d42bb870:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:32:58,917 INFO [RS:1;8124d42bb870:43739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43739 2024-11-10T19:32:58,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8124d42bb870,43739,1731267173934 2024-11-10T19:32:58,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:32:58,947 INFO [RS:1;8124d42bb870:43739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:32:58,955 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8124d42bb870,43739,1731267173934] 2024-11-10T19:32:58,963 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8124d42bb870,43739,1731267173934 already deleted, retry=false 2024-11-10T19:32:58,963 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8124d42bb870,43739,1731267173934 expired; onlineServers=0 2024-11-10T19:32:58,964 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8124d42bb870,46433,1731267172954' ***** 2024-11-10T19:32:58,964 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T19:32:58,964 INFO [M:0;8124d42bb870:46433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:32:58,964 INFO [M:0;8124d42bb870:46433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:32:58,964 DEBUG [M:0;8124d42bb870:46433 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T19:32:58,964 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T19:32:58,964 DEBUG [M:0;8124d42bb870:46433 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T19:32:58,964 DEBUG [master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267175105 {}] cleaner.HFileCleaner(306): Exit Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267175105,5,FailOnTimeoutGroup] 2024-11-10T19:32:58,964 DEBUG [master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267175111 {}] cleaner.HFileCleaner(306): Exit Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267175111,5,FailOnTimeoutGroup] 2024-11-10T19:32:58,964 INFO [M:0;8124d42bb870:46433 {}] hbase.ChoreService(370): Chore service for: master/8124d42bb870:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T19:32:58,964 INFO [M:0;8124d42bb870:46433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:32:58,965 DEBUG [M:0;8124d42bb870:46433 {}] master.HMaster(1795): Stopping service threads 2024-11-10T19:32:58,965 INFO [M:0;8124d42bb870:46433 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T19:32:58,965 INFO [M:0;8124d42bb870:46433 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:32:58,965 INFO [M:0;8124d42bb870:46433 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T19:32:58,965 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T19:32:58,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T19:32:58,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:58,972 DEBUG [M:0;8124d42bb870:46433 {}] zookeeper.ZKUtil(347): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T19:32:58,972 WARN [M:0;8124d42bb870:46433 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T19:32:58,973 INFO [M:0;8124d42bb870:46433 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.lastflushedseqids 2024-11-10T19:32:58,983 INFO [M:0;8124d42bb870:46433 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T19:32:58,983 INFO [M:0;8124d42bb870:46433 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T19:32:58,983 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:32:58,983 INFO [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:58,984 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:58,984 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:32:58,984 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:58,984 INFO [M:0;8124d42bb870:46433 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-10T19:32:58,991 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b649ffc3f98549e6b030f11f59cf849c 2024-11-10T19:32:59,004 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b649ffc3f98549e6b030f11f59cf849c is 82, key is hbase:meta,,1/info:regioninfo/1731267175945/Put/seqid=0 2024-11-10T19:32:59,005 INFO [M:0;8124d42bb870:46433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b649ffc3f98549e6b030f11f59cf849c 2024-11-10T19:32:59,012 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b9868e9e8461441ba5965a86ec5818a5 2024-11-10T19:32:59,023 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9868e9e8461441ba5965a86ec5818a5 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731267176711/Put/seqid=0 2024-11-10T19:32:59,024 INFO [M:0;8124d42bb870:46433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9868e9e8461441ba5965a86ec5818a5 2024-11-10T19:32:59,031 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cd192157885a41e1b7fd8916f68ee1c1 2024-11-10T19:32:59,047 DEBUG [M:0;8124d42bb870:46433 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd192157885a41e1b7fd8916f68ee1c1 is 69, key is 8124d42bb870,43445,1731267173799/rs:state/1731267175126/Put/seqid=0 2024-11-10T19:32:59,048 INFO [M:0;8124d42bb870:46433 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd192157885a41e1b7fd8916f68ee1c1 2024-11-10T19:32:59,051 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b649ffc3f98549e6b030f11f59cf849c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b649ffc3f98549e6b030f11f59cf849c 2024-11-10T19:32:59,053 INFO [M:0;8124d42bb870:46433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b649ffc3f98549e6b030f11f59cf849c, entries=8, sequenceid=84, filesize=5.5 K 2024-11-10T19:32:59,053 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b9868e9e8461441ba5965a86ec5818a5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9868e9e8461441ba5965a86ec5818a5 2024-11-10T19:32:59,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:59,055 INFO [RS:1;8124d42bb870:43739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:32:59,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43739-0x10127862c080002, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:59,055 INFO [M:0;8124d42bb870:46433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b9868e9e8461441ba5965a86ec5818a5, entries=9, sequenceid=84, filesize=6.5 K 2024-11-10T19:32:59,055 INFO [RS:1;8124d42bb870:43739 {}] regionserver.HRegionServer(1031): Exiting; stopping=8124d42bb870,43739,1731267173934; zookeeper connection closed. 2024-11-10T19:32:59,056 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@74739776 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@74739776 2024-11-10T19:32:59,056 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/cd192157885a41e1b7fd8916f68ee1c1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd192157885a41e1b7fd8916f68ee1c1 2024-11-10T19:32:59,056 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T19:32:59,057 INFO [M:0;8124d42bb870:46433 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/cd192157885a41e1b7fd8916f68ee1c1, entries=2, sequenceid=84, filesize=5.1 K 2024-11-10T19:32:59,058 INFO [M:0;8124d42bb870:46433 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false 2024-11-10T19:32:59,060 INFO [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:59,060 DEBUG [M:0;8124d42bb870:46433 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731267178983Disabling compacts and flushes for region at 1731267178983Disabling writes for close at 1731267178984 (+1 ms)Obtaining lock to block concurrent updates at 1731267178984Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731267178984Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731267178984Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731267178985 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731267178985Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731267179003 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731267179003Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731267179006 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731267179023 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731267179023Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731267179026 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731267179047 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731267179047Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e23ce5e: reopening flushed file at 1731267179050 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e6ab957: reopening flushed file at 1731267179053 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c64987f: reopening flushed file at 1731267179056 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false at 1731267179058 (+2 ms)Writing region close event to WAL at 1731267179060 (+2 ms)Closed at 1731267179060 2024-11-10T19:32:59,062 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:32:59,062 INFO [M:0;8124d42bb870:46433 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T19:32:59,062 INFO [M:0;8124d42bb870:46433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46433 2024-11-10T19:32:59,062 INFO [M:0;8124d42bb870:46433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:32:59,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:59,185 INFO [M:0;8124d42bb870:46433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:32:59,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46433-0x10127862c080000, quorum=127.0.0.1:53586, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:32:59,193 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T19:32:59,213 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T19:32:59,213 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,220 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/8124d42bb870:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731267178613 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/8124d42bb870:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731267178781 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/8124d42bb870:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731267178778 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/8124d42bb870:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731267177069 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=162 (was 115) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=15625 (was 15972) 2024-11-10T19:32:59,225 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=162, ProcessCount=11, AvailableMemoryMB=15625 2024-11-10T19:32:59,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/hadoop-log-dir so I do NOT create it in target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd 2024-11-10T19:32:59,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/hadoop-tmp-dir so I do NOT create it in target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd 2024-11-10T19:32:59,226 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,228 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/cluster_7e2f9b7e-4f6a-5995-f067-b02e0742a11c, deleteOnExit=true 2024-11-10T19:32:59,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/cluster_7e2f9b7e-4f6a-5995-f067-b02e0742a11c/zookeeper_0, clientPort=64354, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/cluster_7e2f9b7e-4f6a-5995-f067-b02e0742a11c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/cluster_7e2f9b7e-4f6a-5995-f067-b02e0742a11c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T19:32:59,273 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64354 2024-11-10T19:32:59,288 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 with version=8 2024-11-10T19:32:59,289 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase-staging 2024-11-10T19:32:59,292 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T19:32:59,292 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:59,293 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41271 2024-11-10T19:32:59,295 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41271 connecting to ZooKeeper ensemble=127.0.0.1:64354 2024-11-10T19:32:59,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412710x0, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:59,330 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41271-0x101278645cc0000 connected 2024-11-10T19:32:59,402 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:59,407 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5, hbase.cluster.distributed=false 2024-11-10T19:32:59,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:59,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41271 2024-11-10T19:32:59,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41271 2024-11-10T19:32:59,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41271 2024-11-10T19:32:59,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41271 2024-11-10T19:32:59,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41271 2024-11-10T19:32:59,428 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:59,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,428 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:59,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,429 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:59,429 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:32:59,429 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:59,429 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40003 2024-11-10T19:32:59,431 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40003 connecting to ZooKeeper ensemble=127.0.0.1:64354 2024-11-10T19:32:59,431 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,431 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400030x0, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:59,443 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40003-0x101278645cc0001 connected 2024-11-10T19:32:59,443 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:59,443 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:32:59,444 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache 2024-11-10T19:32:59,444 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T19:32:59,444 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T19:32:59,444 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T19:32:59,444 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T19:32:59,445 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T19:32:59,445 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T19:32:59,445 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:32:59,445 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T19:32:59,445 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence is complete. 2024-11-10T19:32:59,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T19:32:59,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:59,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40003 2024-11-10T19:32:59,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40003 2024-11-10T19:32:59,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40003 2024-11-10T19:32:59,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40003 2024-11-10T19:32:59,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40003 2024-11-10T19:32:59,468 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:32:59,469 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:32:59,470 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45285 2024-11-10T19:32:59,472 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45285 connecting to ZooKeeper ensemble=127.0.0.1:64354 2024-11-10T19:32:59,472 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,472 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:452850x0, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:32:59,485 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:452850x0, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:32:59,485 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45285-0x101278645cc0002 connected 2024-11-10T19:32:59,485 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:32:59,485 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache 2024-11-10T19:32:59,486 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T19:32:59,486 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T19:32:59,486 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T19:32:59,487 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T19:32:59,487 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T19:32:59,487 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T19:32:59,487 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:32:59,487 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T19:32:59,487 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence is complete. 2024-11-10T19:32:59,488 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T19:32:59,489 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:32:59,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-10T19:32:59,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45285 2024-11-10T19:32:59,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45285 2024-11-10T19:32:59,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-10T19:32:59,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45285 2024-11-10T19:32:59,505 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8124d42bb870:41271 2024-11-10T19:32:59,505 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,517 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T19:32:59,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T19:32:59,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,527 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T19:32:59,528 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8124d42bb870,41271,1731267179291 from backup master directory 2024-11-10T19:32:59,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T19:32:59,534 WARN [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:59,534 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,536 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase.id] with ID: f81e89ed-60ce-4f39-9ecb-0e430864be80 2024-11-10T19:32:59,536 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.tmp/hbase.id 2024-11-10T19:32:59,540 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/hbase.id] 2024-11-10T19:32:59,578 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:32:59,578 INFO [master/8124d42bb870:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T19:32:59,584 INFO [master/8124d42bb870:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 6ms. 2024-11-10T19:32:59,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,614 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:32:59,615 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T19:32:59,616 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:59,636 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store 2024-11-10T19:32:59,641 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:32:59,642 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:59,642 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731267179642Disabling compacts and flushes for region at 1731267179642Disabling writes for close at 1731267179642Writing region close event to WAL at 1731267179642Closed at 1731267179642 2024-11-10T19:32:59,642 WARN [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/.initializing 2024-11-10T19:32:59,642 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,647 INFO [master/8124d42bb870:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:32:59,650 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8124d42bb870%2C41271%2C1731267179291, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,41271,1731267179291, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/oldWALs, maxLogs=10 2024-11-10T19:32:59,667 INFO [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/WALs/8124d42bb870,41271,1731267179291/8124d42bb870%2C41271%2C1731267179291.1731267179650 2024-11-10T19:32:59,667 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:32:59,667 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:32:59,667 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:59,667 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,667 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,668 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,672 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T19:32:59,672 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:59,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,676 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T19:32:59,676 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:59,677 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T19:32:59,680 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,680 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:59,681 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,683 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T19:32:59,683 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:32:59,684 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,684 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,684 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,685 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,685 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,685 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:32:59,685 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T19:32:59,691 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:32:59,691 INFO [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72656355, jitterRate=0.08266405761241913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:32:59,692 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731267179668Initializing all the Stores at 1731267179668Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267179668Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267179668Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267179668Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267179668Cleaning up temporary data from old regions at 1731267179685 (+17 ms)Region opened successfully at 1731267179692 (+7 ms) 2024-11-10T19:32:59,692 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T19:32:59,696 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e3cb49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:59,697 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T19:32:59,697 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T19:32:59,697 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T19:32:59,697 INFO [master/8124d42bb870:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T19:32:59,698 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T19:32:59,698 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T19:32:59,698 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T19:32:59,700 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T19:32:59,701 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T19:32:59,724 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T19:32:59,725 INFO [master/8124d42bb870:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T19:32:59,726 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T19:32:59,734 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T19:32:59,734 INFO [master/8124d42bb870:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T19:32:59,735 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T19:32:59,742 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T19:32:59,743 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T19:32:59,750 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T19:32:59,753 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T19:32:59,759 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,767 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,768 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8124d42bb870,41271,1731267179291, sessionid=0x101278645cc0000, setting cluster-up flag (Was=false) 2024-11-10T19:32:59,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,809 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T19:32:59,811 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:32:59,850 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T19:32:59,852 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8124d42bb870,41271,1731267179291 2024-11-10T19:32:59,853 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T19:32:59,854 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:59,854 INFO [master/8124d42bb870:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T19:32:59,854 INFO [master/8124d42bb870:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T19:32:59,855 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8124d42bb870,41271,1731267179291 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T19:32:59,856 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8124d42bb870:0, corePoolSize=5, maxPoolSize=5 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8124d42bb870:0, corePoolSize=10, maxPoolSize=10 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:59,857 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731267209858 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T19:32:59,858 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,858 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:59,859 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T19:32:59,859 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T19:32:59,859 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T19:32:59,859 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T19:32:59,859 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T19:32:59,859 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,859 INFO [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T19:32:59,859 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267179859,5,FailOnTimeoutGroup] 2024-11-10T19:32:59,860 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267179859,5,FailOnTimeoutGroup] 2024-11-10T19:32:59,860 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,859 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T19:32:59,860 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T19:32:59,860 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,860 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,875 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T19:32:59,875 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:59,881 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:32:59,882 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T19:32:59,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:32:59,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:59,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T19:32:59,888 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:32:59,889 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:59,889 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T19:32:59,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:32:59,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:59,892 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T19:32:59,895 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:32:59,895 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:32:59,895 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:32:59,895 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:32:59,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:59,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:32:59,896 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(746): ClusterId : f81e89ed-60ce-4f39-9ecb-0e430864be80 2024-11-10T19:32:59,896 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(746): ClusterId : f81e89ed-60ce-4f39-9ecb-0e430864be80 2024-11-10T19:32:59,896 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:32:59,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:32:59,896 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:32:59,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:32:59,897 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:32:59,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:32:59,904 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:32:59,904 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74420719, jitterRate=0.10895512998104095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:32:59,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731267179882Initializing all the Stores at 1731267179882Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267179882Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267179882Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267179882Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267179882Cleaning up temporary data from old regions at 1731267179896 (+14 ms)Region opened successfully at 1731267179905 (+9 ms) 2024-11-10T19:32:59,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:32:59,905 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:32:59,905 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:32:59,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:32:59,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:32:59,906 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:32:59,906 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731267179905Disabling compacts and flushes for region at 1731267179905Disabling writes for close at 1731267179906 (+1 ms)Writing region close event to WAL at 1731267179906Closed at 1731267179906 2024-11-10T19:32:59,907 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:32:59,908 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T19:32:59,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T19:32:59,909 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:32:59,909 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:32:59,909 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:32:59,909 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:32:59,909 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:32:59,910 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T19:32:59,918 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:32:59,918 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:32:59,919 DEBUG [RS:0;8124d42bb870:40003 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529a16ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:59,919 DEBUG [RS:1;8124d42bb870:45285 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716d4ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:32:59,931 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8124d42bb870:45285 2024-11-10T19:32:59,931 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8124d42bb870:40003 2024-11-10T19:32:59,931 INFO [RS:1;8124d42bb870:45285 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:32:59,931 INFO [RS:1;8124d42bb870:45285 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:32:59,931 INFO [RS:0;8124d42bb870:40003 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:32:59,931 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:32:59,931 INFO [RS:0;8124d42bb870:40003 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:32:59,931 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:32:59,932 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(2659): reportForDuty to master=8124d42bb870,41271,1731267179291 with port=45285, startcode=1731267179468 2024-11-10T19:32:59,932 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(2659): reportForDuty to master=8124d42bb870,41271,1731267179291 with port=40003, startcode=1731267179428 2024-11-10T19:32:59,932 DEBUG [RS:0;8124d42bb870:40003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:32:59,932 DEBUG [RS:1;8124d42bb870:45285 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:32:59,935 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46427, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:32:59,935 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51011, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:32:59,935 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8124d42bb870,45285,1731267179468 2024-11-10T19:32:59,936 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(517): Registering regionserver=8124d42bb870,45285,1731267179468 2024-11-10T19:32:59,937 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8124d42bb870,40003,1731267179428 2024-11-10T19:32:59,937 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(517): Registering regionserver=8124d42bb870,40003,1731267179428 2024-11-10T19:32:59,937 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:59,937 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T19:32:59,937 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:32:59,938 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:32:59,938 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T19:32:59,938 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:32:59,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:32:59,972 DEBUG [RS:1;8124d42bb870:45285 {}] zookeeper.ZKUtil(111): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8124d42bb870,45285,1731267179468 2024-11-10T19:32:59,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8124d42bb870,40003,1731267179428] 2024-11-10T19:32:59,972 WARN [RS:1;8124d42bb870:45285 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:59,972 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8124d42bb870,45285,1731267179468] 2024-11-10T19:32:59,972 INFO [RS:1;8124d42bb870:45285 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:59,973 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,45285,1731267179468 2024-11-10T19:32:59,973 DEBUG [RS:0;8124d42bb870:40003 {}] zookeeper.ZKUtil(111): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8124d42bb870,40003,1731267179428 2024-11-10T19:32:59,973 WARN [RS:0;8124d42bb870:40003 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:32:59,973 INFO [RS:0;8124d42bb870:40003 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:32:59,973 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428 2024-11-10T19:32:59,979 INFO [RS:0;8124d42bb870:40003 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:32:59,979 INFO [RS:1;8124d42bb870:45285 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:32:59,983 INFO [RS:1;8124d42bb870:45285 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:32:59,983 INFO [RS:1;8124d42bb870:45285 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:32:59,983 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,984 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:32:59,985 INFO [RS:1;8124d42bb870:45285 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:32:59,985 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,985 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,985 INFO [RS:0;8124d42bb870:40003 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:32:59,985 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,985 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,985 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:59,986 INFO [RS:0;8124d42bb870:40003 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:59,986 DEBUG [RS:1;8124d42bb870:45285 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:59,987 INFO [RS:0;8124d42bb870:40003 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:32:59,987 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,987 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,987 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,987 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,987 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,45285,1731267179468-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,988 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:32:59,989 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:59,989 DEBUG [RS:0;8124d42bb870:40003 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:32:59,989 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,989 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,989 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,989 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,990 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:32:59,990 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,40003,1731267179428-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:33:00,004 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:33:00,004 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,40003,1731267179428-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,004 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,004 INFO [RS:0;8124d42bb870:40003 {}] regionserver.Replication(171): 8124d42bb870,40003,1731267179428 started 2024-11-10T19:33:00,009 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:33:00,009 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,45285,1731267179468-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,010 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,010 INFO [RS:1;8124d42bb870:45285 {}] regionserver.Replication(171): 8124d42bb870,45285,1731267179468 started 2024-11-10T19:33:00,018 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,018 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1482): Serving as 8124d42bb870,40003,1731267179428, RpcServer on 8124d42bb870/172.17.0.2:40003, sessionid=0x101278645cc0001 2024-11-10T19:33:00,019 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:33:00,019 DEBUG [RS:0;8124d42bb870:40003 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8124d42bb870,40003,1731267179428 2024-11-10T19:33:00,019 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,40003,1731267179428' 2024-11-10T19:33:00,019 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8124d42bb870,40003,1731267179428 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,40003,1731267179428' 2024-11-10T19:33:00,020 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:33:00,021 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:33:00,021 DEBUG [RS:0;8124d42bb870:40003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:33:00,021 INFO [RS:0;8124d42bb870:40003 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:33:00,021 INFO [RS:0;8124d42bb870:40003 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:33:00,028 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,028 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1482): Serving as 8124d42bb870,45285,1731267179468, RpcServer on 8124d42bb870/172.17.0.2:45285, sessionid=0x101278645cc0002 2024-11-10T19:33:00,029 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:33:00,029 DEBUG [RS:1;8124d42bb870:45285 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8124d42bb870,45285,1731267179468 2024-11-10T19:33:00,029 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,45285,1731267179468' 2024-11-10T19:33:00,029 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:33:00,029 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:33:00,030 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:33:00,030 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:33:00,030 DEBUG [RS:1;8124d42bb870:45285 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8124d42bb870,45285,1731267179468 2024-11-10T19:33:00,030 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,45285,1731267179468' 2024-11-10T19:33:00,030 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:33:00,031 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:33:00,031 DEBUG [RS:1;8124d42bb870:45285 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:33:00,031 INFO [RS:1;8124d42bb870:45285 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:33:00,031 INFO [RS:1;8124d42bb870:45285 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:33:00,061 WARN [8124d42bb870:41271 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T19:33:00,122 INFO [RS:0;8124d42bb870:40003 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:33:00,124 INFO [RS:0;8124d42bb870:40003 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C40003%2C1731267179428, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:33:00,132 INFO [RS:1;8124d42bb870:45285 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:33:00,132 INFO [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C45285%2C1731267179468, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,45285,1731267179468, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:33:00,145 INFO [RS:0;8124d42bb870:40003 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428/8124d42bb870%2C40003%2C1731267179428.1731267180128 2024-11-10T19:33:00,145 DEBUG [RS:0;8124d42bb870:40003 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:33:00,149 INFO [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,45285,1731267179468/8124d42bb870%2C45285%2C1731267179468.1731267180134 2024-11-10T19:33:00,149 DEBUG [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:33:00,311 DEBUG [8124d42bb870:41271 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T19:33:00,312 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(204): Hosts are {8124d42bb870=0} racks are {/default-rack=0} 2024-11-10T19:33:00,313 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T19:33:00,313 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T19:33:00,313 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T19:33:00,313 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T19:33:00,313 INFO [8124d42bb870:41271 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T19:33:00,313 INFO [8124d42bb870:41271 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T19:33:00,313 DEBUG [8124d42bb870:41271 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T19:33:00,314 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8124d42bb870,45285,1731267179468 2024-11-10T19:33:00,316 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8124d42bb870,45285,1731267179468, state=OPENING 2024-11-10T19:33:00,342 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T19:33:00,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:00,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:00,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:00,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,352 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T19:33:00,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,352 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8124d42bb870,45285,1731267179468}] 2024-11-10T19:33:00,508 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T19:33:00,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48083, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T19:33:00,517 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T19:33:00,517 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:33:00,517 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T19:33:00,518 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C45285%2C1731267179468.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,45285,1731267179468, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:33:00,537 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,45285,1731267179468/8124d42bb870%2C45285%2C1731267179468.meta.1731267180519.meta 2024-11-10T19:33:00,538 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:33:00,538 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:33:00,538 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T19:33:00,538 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T19:33:00,538 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T19:33:00,538 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T19:33:00,539 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:33:00,539 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T19:33:00,539 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T19:33:00,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T19:33:00,540 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T19:33:00,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:00,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:33:00,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T19:33:00,541 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T19:33:00,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:00,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:33:00,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T19:33:00,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T19:33:00,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:00,542 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:33:00,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T19:33:00,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T19:33:00,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:00,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T19:33:00,543 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T19:33:00,544 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:33:00,544 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740 2024-11-10T19:33:00,545 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T19:33:00,545 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T19:33:00,545 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T19:33:00,545 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T19:33:00,546 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67453493, jitterRate=0.005135372281074524}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T19:33:00,546 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T19:33:00,547 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731267180539Writing region info on filesystem at 1731267180539Initializing all the Stores at 1731267180539Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267180539Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267180539Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267180539Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731267180539Cleaning up temporary data from old regions at 1731267180545 (+6 ms)Running coprocessor post-open hooks at 1731267180546 (+1 ms)Region opened successfully at 1731267180547 (+1 ms) 2024-11-10T19:33:00,547 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731267180507 2024-11-10T19:33:00,550 DEBUG [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T19:33:00,550 INFO [RS_OPEN_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T19:33:00,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8124d42bb870,45285,1731267179468 2024-11-10T19:33:00,551 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8124d42bb870,45285,1731267179468, state=OPEN 2024-11-10T19:33:00,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:33:00,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:33:00,604 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T19:33:00,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,605 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8124d42bb870,45285,1731267179468 2024-11-10T19:33:00,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,605 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T19:33:00,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T19:33:00,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8124d42bb870,45285,1731267179468 in 253 msec 2024-11-10T19:33:00,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T19:33:00,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 701 msec 2024-11-10T19:33:00,615 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T19:33:00,615 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T19:33:00,618 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:33:00,618 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8124d42bb870,45285,1731267179468, seqNum=-1] 2024-11-10T19:33:00,618 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:33:00,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:33:00,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 771 msec 2024-11-10T19:33:00,626 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731267180626, completionTime=-1 2024-11-10T19:33:00,626 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T19:33:00,626 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T19:33:00,628 INFO [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T19:33:00,628 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731267240628 2024-11-10T19:33:00,628 INFO [master/8124d42bb870:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731267300628 2024-11-10T19:33:00,628 INFO [master/8124d42bb870:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T19:33:00,629 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-10T19:33:00,629 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,629 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,630 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,630 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8124d42bb870:41271, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,630 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,630 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,632 DEBUG [master/8124d42bb870:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T19:33:00,634 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.100sec 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:33:00,635 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T19:33:00,636 DEBUG [master/8124d42bb870:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T19:33:00,636 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T19:33:00,636 INFO [master/8124d42bb870:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,41271,1731267179291-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:00,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8223428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:33:00,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8124d42bb870,41271,-1 for getting cluster id 2024-11-10T19:33:00,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T19:33:00,702 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f81e89ed-60ce-4f39-9ecb-0e430864be80' 2024-11-10T19:33:00,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T19:33:00,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f81e89ed-60ce-4f39-9ecb-0e430864be80" 2024-11-10T19:33:00,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d99bdac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:33:00,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8124d42bb870,41271,-1] 2024-11-10T19:33:00,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T19:33:00,705 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:00,707 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T19:33:00,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2ac685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T19:33:00,710 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T19:33:00,712 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8124d42bb870,45285,1731267179468, seqNum=-1] 2024-11-10T19:33:00,712 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:33:00,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55672, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:33:00,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8124d42bb870,41271,1731267179291 2024-11-10T19:33:00,717 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T19:33:00,719 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 8124d42bb870,41271,1731267179291 2024-11-10T19:33:00,719 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c76ec67 2024-11-10T19:33:00,719 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T19:33:00,721 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41434, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T19:33:00,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731267180717', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T19:33:00,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 2024-11-10T19:33:00,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T19:33:00,725 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:00,725 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731267180717" procId is: 4 2024-11-10T19:33:00,726 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T19:33:00,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:33:00,734 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed8e87f79010e3fb9362cef16b074c29, NAME => 'testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731267180717', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1722): Closing ed8e87f79010e3fb9362cef16b074c29, disabling compactions & flushes 2024-11-10T19:33:00,741 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. after waiting 0 ms 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:00,741 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:00,741 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731267180717-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed8e87f79010e3fb9362cef16b074c29: Waiting for close lock at 1731267180741Disabling compacts and flushes for region at 1731267180741Disabling writes for close at 1731267180741Writing region close event to WAL at 1731267180741Closed at 1731267180741 2024-11-10T19:33:00,742 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T19:33:00,743 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731267180742"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731267180742"}]},"ts":"1731267180742"} 2024-11-10T19:33:00,745 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T19:33:00,746 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T19:33:00,747 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731267180717","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731267180746"}]},"ts":"1731267180746"} 2024-11-10T19:33:00,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731267180717, state=ENABLING in hbase:meta 2024-11-10T19:33:00,749 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {8124d42bb870=0} racks are {/default-rack=0} 2024-11-10T19:33:00,750 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T19:33:00,750 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T19:33:00,750 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T19:33:00,750 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T19:33:00,750 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T19:33:00,750 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T19:33:00,750 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T19:33:00,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN}] 2024-11-10T19:33:00,751 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN 2024-11-10T19:33:00,752 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN; state=OFFLINE, location=8124d42bb870,40003,1731267179428; forceNewPlan=false, retain=false 2024-11-10T19:33:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:33:00,903 INFO [8124d42bb870:41271 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T19:33:00,903 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed8e87f79010e3fb9362cef16b074c29, regionState=OPENING, regionLocation=8124d42bb870,40003,1731267179428 2024-11-10T19:33:00,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN because future has completed 2024-11-10T19:33:00,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,40003,1731267179428}] 2024-11-10T19:33:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:33:01,063 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T19:33:01,067 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53007, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T19:33:01,071 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:01,071 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ed8e87f79010e3fb9362cef16b074c29, NAME => 'testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:33:01,072 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731267180717 ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,072 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:33:01,072 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,072 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,075 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,080 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed8e87f79010e3fb9362cef16b074c29 columnFamilyName family 2024-11-10T19:33:01,080 DEBUG [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:01,080 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] regionserver.HStore(327): Store=ed8e87f79010e3fb9362cef16b074c29/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:33:01,080 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,081 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,081 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,081 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,081 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,082 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,087 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T19:33:01,087 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ed8e87f79010e3fb9362cef16b074c29; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71696145, jitterRate=0.06835581362247467}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T19:33:01,087 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:01,088 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ed8e87f79010e3fb9362cef16b074c29: Running coprocessor pre-open hook at 1731267181074Writing region info on filesystem at 1731267181074Initializing all the Stores at 1731267181074Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267181075 (+1 ms)Cleaning up temporary data from old regions at 1731267181081 (+6 ms)Running coprocessor post-open hooks at 1731267181087 (+6 ms)Region opened successfully at 1731267181088 (+1 ms) 2024-11-10T19:33:01,089 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29., pid=6, masterSystemTime=1731267181062 2024-11-10T19:33:01,091 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:01,091 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:01,091 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed8e87f79010e3fb9362cef16b074c29, regionState=OPEN, openSeqNum=2, regionLocation=8124d42bb870,40003,1731267179428 2024-11-10T19:33:01,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,40003,1731267179428 because future has completed 2024-11-10T19:33:01,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T19:33:01,098 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,40003,1731267179428 in 189 msec 2024-11-10T19:33:01,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T19:33:01,100 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN in 348 msec 2024-11-10T19:33:01,101 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T19:33:01,101 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731267180717","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731267181101"}]},"ts":"1731267181101"} 2024-11-10T19:33:01,104 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731267180717, state=ENABLED in hbase:meta 2024-11-10T19:33:01,105 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T19:33:01,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731267180717 in 383 msec 2024-11-10T19:33:01,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41271 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T19:33:01,354 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731267180717 completed 2024-11-10T19:33:01,354 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731267180717 get assigned. Timeout = 60000ms 2024-11-10T19:33:01,355 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T19:33:01,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731267180717 assigned to meta. Checking AM states. 2024-11-10T19:33:01,363 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T19:33:01,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731267180717 assigned. 2024-11-10T19:33:01,369 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731267180717', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29., hostname=8124d42bb870,40003,1731267179428, seqNum=2] 2024-11-10T19:33:01,369 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T19:33:01,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T19:33:01,375 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ed8e87f79010e3fb9362cef16b074c29 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T19:33:01,383 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:01,396 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/.tmp/family/a2d7c80cc3504176b94804d15fb8da2a is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T19:33:01,397 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/.tmp/family/a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:01,399 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/.tmp/family/a2d7c80cc3504176b94804d15fb8da2a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:01,400 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, delay=984 ms 2024-11-10T19:33:01,402 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T19:33:01,402 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ed8e87f79010e3fb9362cef16b074c29 in 27ms, sequenceid=6, compaction requested=false 2024-11-10T19:33:01,402 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731267180717' 2024-11-10T19:33:01,403 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ed8e87f79010e3fb9362cef16b074c29: 2024-11-10T19:33:01,421 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T19:33:02,387 DEBUG [hfile-prefetch-1731267178613 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a2d7c80cc3504176b94804d15fb8da2a_0. 2024-11-10T19:33:02,388 DEBUG [hfile-prefetch-1731267178613 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key a2d7c80cc3504176b94804d15fb8da2a_117. 2024-11-10T19:33:02,389 DEBUG [hfile-prefetch-1731267178613 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, with total blocks 2, and data blocks 1 2024-11-10T19:33:02,389 DEBUG [hfile-prefetch-1731267178613 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T19:33:02,389 DEBUG [hfile-prefetch-1731267178613 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T19:33:02,390 DEBUG [hfile-prefetch-1731267178613 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a. Will sleep 100ms and try the verification again. 2024-11-10T19:33:02,403 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;8124d42bb870:40003,5,FailOnTimeoutGroup] 2024-11-10T19:33:02,403 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8124d42bb870,40003,1731267179428' ***** 2024-11-10T19:33:02,403 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-10T19:33:02,404 INFO [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T19:33:02,469 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T19:33:02,469 INFO [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence 2024-11-10T19:33:02,469 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T19:33:02,469 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T19:33:02,470 INFO [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T19:33:02,470 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T19:33:02,480 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1488): Thread RS:0;8124d42bb870:40003 started persisting bucket cache to file 2024-11-10T19:33:02,480 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:33:02,480 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:33:02,483 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:33:02,484 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267182480 2024-11-10T19:33:02,484 DEBUG [RS:0;8124d42bb870:40003 {}] bucket.BucketCache(1510): Thread RS:0;8124d42bb870:40003 finished persisting bucket cache to file, renaming 2024-11-10T19:33:02,484 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:33:02,484 INFO [RS:0;8124d42bb870:40003 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:33:02,484 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:33:02,484 INFO [RS:0;8124d42bb870:40003 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:33:02,484 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(3091): Received CLOSE for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:02,485 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(959): stopping server 8124d42bb870,40003,1731267179428 2024-11-10T19:33:02,485 INFO [RS:0;8124d42bb870:40003 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:33:02,485 INFO [RS:0;8124d42bb870:40003 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8124d42bb870:40003. 2024-11-10T19:33:02,485 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T19:33:02,485 DEBUG [RS:0;8124d42bb870:40003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:33:02,485 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed8e87f79010e3fb9362cef16b074c29, disabling compactions & flushes 2024-11-10T19:33:02,485 DEBUG [RS:0;8124d42bb870:40003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:02,485 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:02,485 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:02,486 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T19:33:02,486 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. after waiting 0 ms 2024-11-10T19:33:02,486 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1325): Online Regions={ed8e87f79010e3fb9362cef16b074c29=testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.} 2024-11-10T19:33:02,486 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:02,486 DEBUG [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1351): Waiting on ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:02,486 DEBUG [StoreFileCloser-ed8e87f79010e3fb9362cef16b074c29-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:02,486 WARN [hfile-prefetch-1731267178613 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T19:33:02,487 DEBUG [hfile-prefetch-1731267178613 {}] hfile.PrefetchExecutor(121): Prefetch completed for a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:02,489 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T19:33:02,490 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T19:33:02,490 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:33:02,490 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:33:02,492 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T19:33:02,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:33:02,492 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:02,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267182490 2024-11-10T19:33:02,492 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T19:33:02,492 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed8e87f79010e3fb9362cef16b074c29: Waiting for close lock at 1731267182485Running coprocessor pre-close hooks at 1731267182485Disabling compacts and flushes for region at 1731267182485Disabling writes for close at 1731267182486 (+1 ms)Writing region close event to WAL at 1731267182487 (+1 ms)Running coprocessor post-close hooks at 1731267182492 (+5 ms)Closed at 1731267182492 2024-11-10T19:33:02,492 INFO [regionserver/8124d42bb870:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:02,493 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:02,686 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(976): stopping server 8124d42bb870,40003,1731267179428; all regions closed. 2024-11-10T19:33:02,688 DEBUG [RS:0;8124d42bb870:40003 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C40003%2C1731267179428:(num 1731267180128) 2024-11-10T19:33:02,689 DEBUG [RS:0;8124d42bb870:40003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] hbase.ChoreService(370): Chore service for: regionserver/8124d42bb870:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:33:02,689 INFO [regionserver/8124d42bb870:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:33:02,689 INFO [RS:0;8124d42bb870:40003 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:33:02,690 INFO [RS:0;8124d42bb870:40003 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:33:02,690 INFO [RS:0;8124d42bb870:40003 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:33:02,690 INFO [RS:0;8124d42bb870:40003 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40003 2024-11-10T19:33:02,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:33:02,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8124d42bb870,40003,1731267179428 2024-11-10T19:33:02,726 INFO [RS:0;8124d42bb870:40003 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:33:02,734 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8124d42bb870,40003,1731267179428] 2024-11-10T19:33:02,742 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8124d42bb870,40003,1731267179428 already deleted, retry=false 2024-11-10T19:33:02,742 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 8124d42bb870,40003,1731267179428 on 8124d42bb870,41271,1731267179291 2024-11-10T19:33:02,746 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 8124d42bb870,40003,1731267179428, splitWal=true, meta=false 2024-11-10T19:33:02,748 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 8124d42bb870,40003,1731267179428 (carryingMeta=false) 8124d42bb870,40003,1731267179428/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@43e684d8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-10T19:33:02,749 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 8124d42bb870,40003,1731267179428, splitWal=true, meta=false 2024-11-10T19:33:02,750 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 8124d42bb870,40003,1731267179428 had 1 regions 2024-11-10T19:33:02,752 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 8124d42bb870,40003,1731267179428, splitWal=true, meta=false, isMeta: false 2024-11-10T19:33:02,752 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428-splitting 2024-11-10T19:33:02,752 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428-splitting dir is empty, no logs to split. 2024-11-10T19:33:02,753 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8124d42bb870,40003,1731267179428 WAL count=0, meta=false 2024-11-10T19:33:02,755 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428-splitting dir is empty, no logs to split. 2024-11-10T19:33:02,755 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8124d42bb870,40003,1731267179428 WAL count=0, meta=false 2024-11-10T19:33:02,755 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 8124d42bb870,40003,1731267179428 WAL splitting is done? wals=0, meta=false 2024-11-10T19:33:02,755 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 8124d42bb870,40003,1731267179428 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,40003,1731267179428-splitting does not exist 2024-11-10T19:33:02,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN}] 2024-11-10T19:33:02,759 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN 2024-11-10T19:33:02,760 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-10T19:33:02,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:02,834 INFO [RS:0;8124d42bb870:40003 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:33:02,834 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40003-0x101278645cc0001, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:02,834 INFO [RS:0;8124d42bb870:40003 {}] regionserver.HRegionServer(1031): Exiting; stopping=8124d42bb870,40003,1731267179428; zookeeper connection closed. 2024-11-10T19:33:02,835 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@761c6155 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@761c6155 2024-11-10T19:33:02,911 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ed8e87f79010e3fb9362cef16b074c29, regionState=OPENING, regionLocation=8124d42bb870,45285,1731267179468 2024-11-10T19:33:02,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN because future has completed 2024-11-10T19:33:02,916 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,45285,1731267179468}] 2024-11-10T19:33:02,925 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8124d42bb870:0 server-side Connection retries=45 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T19:33:02,925 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T19:33:02,926 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46645 2024-11-10T19:33:02,927 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46645 connecting to ZooKeeper ensemble=127.0.0.1:64354 2024-11-10T19:33:02,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:33:02,928 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:33:02,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466450x0, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T19:33:02,958 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46645-0x101278645cc0003 connected 2024-11-10T19:33:02,958 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T19:33:02,958 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T19:33:02,959 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T19:33:02,960 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache 2024-11-10T19:33:02,960 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T19:33:02,960 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T19:33:02,960 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T19:33:02,961 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T19:33:02,962 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T19:33:02,962 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-10T19:33:02,962 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T19:33:02,963 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-10T19:33:02,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T19:33:02,966 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T19:33:02,966 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-10T19:33:02,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46645 2024-11-10T19:33:02,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46645 2024-11-10T19:33:02,967 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-10T19:33:02,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-10T19:33:02,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46645 2024-11-10T19:33:02,968 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T19:33:02,968 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-10T19:33:02,968 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence is complete. 2024-11-10T19:33:02,969 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(746): ClusterId : f81e89ed-60ce-4f39-9ecb-0e430864be80 2024-11-10T19:33:02,969 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T19:33:02,976 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T19:33:02,976 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T19:33:02,985 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T19:33:02,986 DEBUG [RS:2;8124d42bb870:46645 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3daaf78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8124d42bb870/172.17.0.2:0 2024-11-10T19:33:03,005 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;8124d42bb870:46645 2024-11-10T19:33:03,005 INFO [RS:2;8124d42bb870:46645 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T19:33:03,005 INFO [RS:2;8124d42bb870:46645 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T19:33:03,005 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T19:33:03,006 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(2659): reportForDuty to master=8124d42bb870,41271,1731267179291 with port=46645, startcode=1731267182925 2024-11-10T19:33:03,007 DEBUG [RS:2;8124d42bb870:46645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T19:33:03,009 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54509, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T19:33:03,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,009 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41271 {}] master.ServerManager(517): Registering regionserver=8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,011 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5 2024-11-10T19:33:03,011 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T19:33:03,011 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T19:33:03,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:33:03,018 DEBUG [RS:2;8124d42bb870:46645 {}] zookeeper.ZKUtil(111): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,018 WARN [RS:2;8124d42bb870:46645 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T19:33:03,018 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8124d42bb870,46645,1731267182925] 2024-11-10T19:33:03,018 INFO [RS:2;8124d42bb870:46645 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T19:33:03,018 DEBUG [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,023 INFO [RS:2;8124d42bb870:46645 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T19:33:03,026 INFO [RS:2;8124d42bb870:46645 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T19:33:03,027 INFO [RS:2;8124d42bb870:46645 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T19:33:03,027 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,027 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T19:33:03,027 INFO [RS:2;8124d42bb870:46645 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T19:33:03,027 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8124d42bb870:0, corePoolSize=2, maxPoolSize=2 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8124d42bb870:0, corePoolSize=1, maxPoolSize=1 2024-11-10T19:33:03,028 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:33:03,029 DEBUG [RS:2;8124d42bb870:46645 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8124d42bb870:0, corePoolSize=3, maxPoolSize=3 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,029 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46645,1731267182925-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T19:33:03,047 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T19:33:03,047 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=8124d42bb870,46645,1731267182925-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,047 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,047 INFO [RS:2;8124d42bb870:46645 {}] regionserver.Replication(171): 8124d42bb870,46645,1731267182925 started 2024-11-10T19:33:03,063 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T19:33:03,063 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1482): Serving as 8124d42bb870,46645,1731267182925, RpcServer on 8124d42bb870/172.17.0.2:46645, sessionid=0x101278645cc0003 2024-11-10T19:33:03,064 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T19:33:03,064 DEBUG [RS:2;8124d42bb870:46645 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,064 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,46645,1731267182925' 2024-11-10T19:33:03,064 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8124d42bb870,46645,1731267182925' 2024-11-10T19:33:03,065 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T19:33:03,066 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T19:33:03,066 DEBUG [RS:2;8124d42bb870:46645 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T19:33:03,066 INFO [RS:2;8124d42bb870:46645 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T19:33:03,066 INFO [RS:2;8124d42bb870:46645 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T19:33:03,070 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,070 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ed8e87f79010e3fb9362cef16b074c29, NAME => 'testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.', STARTKEY => '', ENDKEY => ''} 2024-11-10T19:33:03,071 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731267180717 ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,071 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T19:33:03,071 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,071 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,072 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,073 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed8e87f79010e3fb9362cef16b074c29 columnFamilyName family 2024-11-10T19:33:03,073 DEBUG [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T19:33:03,075 DEBUG [StoreFileOpener-ed8e87f79010e3fb9362cef16b074c29-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a, delay=940 ms 2024-11-10T19:33:03,077 DEBUG [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:03,077 INFO [StoreOpener-ed8e87f79010e3fb9362cef16b074c29-1 {}] regionserver.HStore(327): Store=ed8e87f79010e3fb9362cef16b074c29/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T19:33:03,077 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,078 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,078 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,078 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,078 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,080 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,080 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ed8e87f79010e3fb9362cef16b074c29; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59637683, jitterRate=-0.11132927238941193}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T19:33:03,080 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,081 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ed8e87f79010e3fb9362cef16b074c29: Running coprocessor pre-open hook at 1731267183072Writing region info on filesystem at 1731267183072Initializing all the Stores at 1731267183072Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731267183072Cleaning up temporary data from old regions at 1731267183079 (+7 ms)Running coprocessor post-open hooks at 1731267183080 (+1 ms)Region opened successfully at 1731267183081 (+1 ms) 2024-11-10T19:33:03,082 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29., pid=9, masterSystemTime=1731267183068 2024-11-10T19:33:03,085 DEBUG [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,085 INFO [RS_OPEN_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,085 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ed8e87f79010e3fb9362cef16b074c29, regionState=OPEN, openSeqNum=10, regionLocation=8124d42bb870,45285,1731267179468 2024-11-10T19:33:03,088 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,45285,1731267179468 because future has completed 2024-11-10T19:33:03,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-10T19:33:03,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ed8e87f79010e3fb9362cef16b074c29, server=8124d42bb870,45285,1731267179468 in 173 msec 2024-11-10T19:33:03,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T19:33:03,094 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 8124d42bb870,40003,1731267179428 after splitting done 2024-11-10T19:33:03,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731267180717, region=ed8e87f79010e3fb9362cef16b074c29, ASSIGN in 334 msec 2024-11-10T19:33:03,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 8124d42bb870,40003,1731267179428, splitWal=true, meta=false in 351 msec 2024-11-10T19:33:03,167 INFO [RS:2;8124d42bb870:46645 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T19:33:03,168 INFO [RS:2;8124d42bb870:46645 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=8124d42bb870%2C46645%2C1731267182925, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,46645,1731267182925, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs, maxLogs=55 2024-11-10T19:33:03,186 INFO [RS:2;8124d42bb870:46645 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/WALs/8124d42bb870,46645,1731267182925/8124d42bb870%2C46645%2C1731267182925.1731267183169 2024-11-10T19:33:03,186 DEBUG [RS:2;8124d42bb870:46645 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T19:33:03,401 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T19:33:03,402 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T19:33:03,405 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731267180717 2024-11-10T19:33:03,405 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731267180717 Metrics about Tables on a single HBase RegionServer 2024-11-10T19:33:03,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T19:33:03,407 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T19:33:03,407 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T19:33:03,407 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T19:33:03,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T19:33:03,564 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:33:03,565 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:33:03,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,565 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T19:33:03,566 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T19:33:03,566 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1603238852, stopped=false 2024-11-10T19:33:03,566 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8124d42bb870,41271,1731267179291 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:03,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:33:03,617 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:03,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T19:33:03,618 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:33:03,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8124d42bb870,45285,1731267179468' ***** 2024-11-10T19:33:03,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:33:03,618 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:33:03,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T19:33:03,618 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8124d42bb870,46645,1731267182925' ***** 2024-11-10T19:33:03,618 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T19:33:03,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T19:33:03,619 INFO [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T19:33:03,619 INFO [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T19:33:03,619 INFO [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence 2024-11-10T19:33:03,619 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T19:33:03,619 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T19:33:03,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T19:33:03,619 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T19:33:03,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T19:33:03,619 INFO [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T19:33:03,620 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T19:33:03,620 INFO [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/588e2863-bd30-044a-ced3-03d524eac4dd/bucket.persistence 2024-11-10T19:33:03,620 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T19:33:03,620 INFO [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T19:33:03,620 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T19:33:03,630 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:33:03,630 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1488): Thread RS:2;8124d42bb870:46645 started persisting bucket cache to file 2024-11-10T19:33:03,630 INFO [regionserver/8124d42bb870:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:03,630 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1488): Thread RS:1;8124d42bb870:45285 started persisting bucket cache to file 2024-11-10T19:33:03,630 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:33:03,630 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:33:03,630 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T19:33:03,630 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T19:33:03,634 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:33:03,634 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T19:33:03,634 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267183630 2024-11-10T19:33:03,634 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731267183630 2024-11-10T19:33:03,634 DEBUG [RS:1;8124d42bb870:45285 {}] bucket.BucketCache(1510): Thread RS:1;8124d42bb870:45285 finished persisting bucket cache to file, renaming 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:33:03,635 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(3091): Received CLOSE for ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(959): stopping server 8124d42bb870,45285,1731267179468 2024-11-10T19:33:03,635 INFO [RS:1;8124d42bb870:45285 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:33:03,636 INFO [RS:1;8124d42bb870:45285 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8124d42bb870:45285. 2024-11-10T19:33:03,636 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T19:33:03,636 DEBUG [RS:1;8124d42bb870:45285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:33:03,636 DEBUG [RS:1;8124d42bb870:45285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,636 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed8e87f79010e3fb9362cef16b074c29, disabling compactions & flushes 2024-11-10T19:33:03,636 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,636 INFO [RS:1;8124d42bb870:45285 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:33:03,636 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,636 INFO [RS:1;8124d42bb870:45285 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:33:03,636 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. after waiting 0 ms 2024-11-10T19:33:03,636 INFO [RS:1;8124d42bb870:45285 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:33:03,636 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,636 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T19:33:03,637 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T19:33:03,637 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, ed8e87f79010e3fb9362cef16b074c29=testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29.} 2024-11-10T19:33:03,637 DEBUG [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ed8e87f79010e3fb9362cef16b074c29 2024-11-10T19:33:03,637 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T19:33:03,637 DEBUG [StoreFileCloser-ed8e87f79010e3fb9362cef16b074c29-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/family/a2d7c80cc3504176b94804d15fb8da2a 2024-11-10T19:33:03,637 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T19:33:03,637 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T19:33:03,637 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T19:33:03,637 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T19:33:03,637 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T19:33:03,638 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-10T19:33:03,643 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/default/testBlockEvictionOnGracefulStop1731267180717/ed8e87f79010e3fb9362cef16b074c29/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T19:33:03,644 INFO [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,644 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed8e87f79010e3fb9362cef16b074c29: Waiting for close lock at 1731267183636Running coprocessor pre-close hooks at 1731267183636Disabling compacts and flushes for region at 1731267183636Disabling writes for close at 1731267183636Writing region close event to WAL at 1731267183637 (+1 ms)Running coprocessor post-close hooks at 1731267183643 (+6 ms)Closed at 1731267183644 (+1 ms) 2024-11-10T19:33:03,644 DEBUG [RS_CLOSE_REGION-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29. 2024-11-10T19:33:03,645 DEBUG [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1510): Thread RS:2;8124d42bb870:46645 finished persisting bucket cache to file, renaming 2024-11-10T19:33:03,645 WARN [RS:2;8124d42bb870:46645 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(959): stopping server 8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:33:03,645 INFO [RS:2;8124d42bb870:46645 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;8124d42bb870:46645. 2024-11-10T19:33:03,645 DEBUG [RS:2;8124d42bb870:46645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T19:33:03,646 DEBUG [RS:2;8124d42bb870:46645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,646 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(976): stopping server 8124d42bb870,46645,1731267182925; all regions closed. 2024-11-10T19:33:03,647 DEBUG [RS:2;8124d42bb870:46645 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:33:03,647 INFO [RS:2;8124d42bb870:46645 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C46645%2C1731267182925:(num 1731267183169) 2024-11-10T19:33:03,648 DEBUG [RS:2;8124d42bb870:46645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] hbase.ChoreService(370): Chore service for: regionserver/8124d42bb870:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T19:33:03,648 INFO [regionserver/8124d42bb870:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:33:03,648 INFO [RS:2;8124d42bb870:46645 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46645 2024-11-10T19:33:03,650 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b387774781b345e28c03a4060dcd23cc 2024-11-10T19:33:03,666 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/b387774781b345e28c03a4060dcd23cc is 209, key is testBlockEvictionOnGracefulStop1731267180717,,1731267180722.ed8e87f79010e3fb9362cef16b074c29./info:regioninfo/1731267183085/Put/seqid=0 2024-11-10T19:33:03,667 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/b387774781b345e28c03a4060dcd23cc 2024-11-10T19:33:03,673 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fa9b97249aac4343af638bed2ac0940a 2024-11-10T19:33:03,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:33:03,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8124d42bb870,46645,1731267182925 2024-11-10T19:33:03,675 INFO [RS:2;8124d42bb870:46645 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:33:03,684 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8124d42bb870,46645,1731267182925] 2024-11-10T19:33:03,687 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/fa9b97249aac4343af638bed2ac0940a is 43, key is default/ns:d/1731267180621/Put/seqid=0 2024-11-10T19:33:03,688 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/fa9b97249aac4343af638bed2ac0940a 2024-11-10T19:33:03,692 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8124d42bb870,46645,1731267182925 already deleted, retry=false 2024-11-10T19:33:03,692 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8124d42bb870,46645,1731267182925 expired; onlineServers=1 2024-11-10T19:33:03,692 INFO [regionserver/8124d42bb870:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:03,695 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ef19504008fd456093e4e28bdba20730 2024-11-10T19:33:03,708 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/ef19504008fd456093e4e28bdba20730 is 80, key is testBlockEvictionOnGracefulStop1731267180717/table:state/1731267181101/Put/seqid=0 2024-11-10T19:33:03,708 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/ef19504008fd456093e4e28bdba20730 2024-11-10T19:33:03,710 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/info/b387774781b345e28c03a4060dcd23cc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/b387774781b345e28c03a4060dcd23cc 2024-11-10T19:33:03,712 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/b387774781b345e28c03a4060dcd23cc, delay=937 ms 2024-11-10T19:33:03,713 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/b387774781b345e28c03a4060dcd23cc, entries=14, sequenceid=13, filesize=7.5 K 2024-11-10T19:33:03,713 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/ns/fa9b97249aac4343af638bed2ac0940a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/fa9b97249aac4343af638bed2ac0940a 2024-11-10T19:33:03,714 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/fa9b97249aac4343af638bed2ac0940a, delay=936 ms 2024-11-10T19:33:03,715 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/fa9b97249aac4343af638bed2ac0940a, entries=2, sequenceid=13, filesize=5.0 K 2024-11-10T19:33:03,715 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/.tmp/table/ef19504008fd456093e4e28bdba20730 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/ef19504008fd456093e4e28bdba20730 2024-11-10T19:33:03,716 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/ef19504008fd456093e4e28bdba20730, delay=960 ms 2024-11-10T19:33:03,717 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/ef19504008fd456093e4e28bdba20730, entries=2, sequenceid=13, filesize=5.3 K 2024-11-10T19:33:03,718 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false 2024-11-10T19:33:03,719 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/info/b387774781b345e28c03a4060dcd23cc 2024-11-10T19:33:03,720 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/ns/fa9b97249aac4343af638bed2ac0940a 2024-11-10T19:33:03,720 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/table/ef19504008fd456093e4e28bdba20730 2024-11-10T19:33:03,724 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-10T19:33:03,724 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T19:33:03,724 INFO [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T19:33:03,724 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731267183637Running coprocessor pre-close hooks at 1731267183637Disabling compacts and flushes for region at 1731267183637Disabling writes for close at 1731267183637Obtaining lock to block concurrent updates at 1731267183638 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731267183638Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731267183638Flushing stores of hbase:meta,,1.1588230740 at 1731267183639 (+1 ms)Flushing 1588230740/info: creating writer at 1731267183639Flushing 1588230740/info: appending metadata at 1731267183666 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731267183666Flushing 1588230740/ns: creating writer at 1731267183669 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731267183687 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731267183687Flushing 1588230740/table: creating writer at 1731267183690 (+3 ms)Flushing 1588230740/table: appending metadata at 1731267183707 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731267183707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3231331a: reopening flushed file at 1731267183710 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f6ea58: reopening flushed file at 1731267183713 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@349639f1: reopening flushed file at 1731267183715 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false at 1731267183718 (+3 ms)Writing region close event to WAL at 1731267183720 (+2 ms)Running coprocessor post-close hooks at 1731267183724 (+4 ms)Closed at 1731267183724 2024-11-10T19:33:03,725 DEBUG [RS_CLOSE_META-regionserver/8124d42bb870:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T19:33:03,784 INFO [RS:2;8124d42bb870:46645 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:33:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:03,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46645-0x101278645cc0003, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:03,784 INFO [RS:2;8124d42bb870:46645 {}] regionserver.HRegionServer(1031): Exiting; stopping=8124d42bb870,46645,1731267182925; zookeeper connection closed. 2024-11-10T19:33:03,784 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ecfe31 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ecfe31 2024-11-10T19:33:03,837 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(976): stopping server 8124d42bb870,45285,1731267179468; all regions closed. 2024-11-10T19:33:03,841 DEBUG [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:33:03,842 INFO [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C45285%2C1731267179468.meta:.meta(num 1731267180519) 2024-11-10T19:33:03,846 DEBUG [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/oldWALs 2024-11-10T19:33:03,846 INFO [RS:1;8124d42bb870:45285 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8124d42bb870%2C45285%2C1731267179468:(num 1731267180134) 2024-11-10T19:33:03,846 DEBUG [RS:1;8124d42bb870:45285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T19:33:03,846 INFO [RS:1;8124d42bb870:45285 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T19:33:03,847 INFO [RS:1;8124d42bb870:45285 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:33:03,847 INFO [RS:1;8124d42bb870:45285 {}] hbase.ChoreService(370): Chore service for: regionserver/8124d42bb870:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T19:33:03,848 INFO [RS:1;8124d42bb870:45285 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:33:03,848 INFO [regionserver/8124d42bb870:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:33:03,848 INFO [RS:1;8124d42bb870:45285 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45285 2024-11-10T19:33:03,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8124d42bb870,45285,1731267179468 2024-11-10T19:33:03,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T19:33:03,858 INFO [RS:1;8124d42bb870:45285 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:33:03,867 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8124d42bb870,45285,1731267179468] 2024-11-10T19:33:03,875 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8124d42bb870,45285,1731267179468 already deleted, retry=false 2024-11-10T19:33:03,875 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8124d42bb870,45285,1731267179468 expired; onlineServers=0 2024-11-10T19:33:03,875 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8124d42bb870,41271,1731267179291' ***** 2024-11-10T19:33:03,875 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T19:33:03,875 INFO [M:0;8124d42bb870:41271 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T19:33:03,875 INFO [M:0;8124d42bb870:41271 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T19:33:03,875 DEBUG [M:0;8124d42bb870:41271 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T19:33:03,876 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T19:33:03,876 DEBUG [M:0;8124d42bb870:41271 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T19:33:03,876 DEBUG [master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267179859 {}] cleaner.HFileCleaner(306): Exit Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.large.0-1731267179859,5,FailOnTimeoutGroup] 2024-11-10T19:33:03,876 DEBUG [master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267179859 {}] cleaner.HFileCleaner(306): Exit Thread[master/8124d42bb870:0:becomeActiveMaster-HFileCleaner.small.0-1731267179859,5,FailOnTimeoutGroup] 2024-11-10T19:33:03,876 INFO [M:0;8124d42bb870:41271 {}] hbase.ChoreService(370): Chore service for: master/8124d42bb870:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T19:33:03,876 INFO [M:0;8124d42bb870:41271 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T19:33:03,876 DEBUG [M:0;8124d42bb870:41271 {}] master.HMaster(1795): Stopping service threads 2024-11-10T19:33:03,876 INFO [M:0;8124d42bb870:41271 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T19:33:03,876 INFO [M:0;8124d42bb870:41271 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T19:33:03,877 INFO [M:0;8124d42bb870:41271 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T19:33:03,877 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T19:33:03,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T19:33:03,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T19:33:03,884 DEBUG [M:0;8124d42bb870:41271 {}] zookeeper.ZKUtil(347): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T19:33:03,884 WARN [M:0;8124d42bb870:41271 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T19:33:03,884 INFO [M:0;8124d42bb870:41271 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/.lastflushedseqids 2024-11-10T19:33:03,889 INFO [M:0;8124d42bb870:41271 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T19:33:03,890 INFO [M:0;8124d42bb870:41271 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T19:33:03,890 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T19:33:03,890 INFO [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:33:03,890 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:33:03,890 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T19:33:03,890 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:33:03,890 INFO [M:0;8124d42bb870:41271 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-10T19:33:03,898 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e77e03ac6eb44b929f72cfe9ee648f89 2024-11-10T19:33:03,910 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e77e03ac6eb44b929f72cfe9ee648f89 is 82, key is hbase:meta,,1/info:regioninfo/1731267180550/Put/seqid=0 2024-11-10T19:33:03,911 INFO [M:0;8124d42bb870:41271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e77e03ac6eb44b929f72cfe9ee648f89 2024-11-10T19:33:03,917 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2a2a7aca0b4642fea32c513708f13ca8 2024-11-10T19:33:03,930 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a2a7aca0b4642fea32c513708f13ca8 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731267181106/Put/seqid=0 2024-11-10T19:33:03,931 INFO [M:0;8124d42bb870:41271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a2a7aca0b4642fea32c513708f13ca8 2024-11-10T19:33:03,937 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fa9cb4c7e6c14204a74bf34c10fe35f1 2024-11-10T19:33:03,950 DEBUG [M:0;8124d42bb870:41271 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa9cb4c7e6c14204a74bf34c10fe35f1 is 69, key is 8124d42bb870,45285,1731267179468/rs:state/1731267179936/Put/seqid=0 2024-11-10T19:33:03,950 INFO [M:0;8124d42bb870:41271 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa9cb4c7e6c14204a74bf34c10fe35f1 2024-11-10T19:33:03,952 INFO [M:0;8124d42bb870:41271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa9cb4c7e6c14204a74bf34c10fe35f1 2024-11-10T19:33:03,952 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e77e03ac6eb44b929f72cfe9ee648f89 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e77e03ac6eb44b929f72cfe9ee648f89 2024-11-10T19:33:03,954 INFO [M:0;8124d42bb870:41271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e77e03ac6eb44b929f72cfe9ee648f89, entries=8, sequenceid=83, filesize=5.5 K 2024-11-10T19:33:03,954 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/2a2a7aca0b4642fea32c513708f13ca8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2a2a7aca0b4642fea32c513708f13ca8 2024-11-10T19:33:03,956 INFO [M:0;8124d42bb870:41271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/2a2a7aca0b4642fea32c513708f13ca8, entries=9, sequenceid=83, filesize=6.4 K 2024-11-10T19:33:03,956 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fa9cb4c7e6c14204a74bf34c10fe35f1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa9cb4c7e6c14204a74bf34c10fe35f1 2024-11-10T19:33:03,957 INFO [M:0;8124d42bb870:41271 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa9cb4c7e6c14204a74bf34c10fe35f1 2024-11-10T19:33:03,958 INFO [M:0;8124d42bb870:41271 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/7a107346-4015-9fed-4501-b331d852e49f/3326da17-1fd2-c039-e4d9-ab3f553a55d5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fa9cb4c7e6c14204a74bf34c10fe35f1, entries=3, sequenceid=83, filesize=5.3 K 2024-11-10T19:33:03,958 INFO [M:0;8124d42bb870:41271 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false 2024-11-10T19:33:03,960 INFO [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T19:33:03,960 DEBUG [M:0;8124d42bb870:41271 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731267183890Disabling compacts and flushes for region at 1731267183890Disabling writes for close at 1731267183890Obtaining lock to block concurrent updates at 1731267183890Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731267183890Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1731267183890Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731267183891 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731267183891Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731267183910 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731267183910Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731267183912 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731267183929 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731267183930 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731267183932 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731267183949 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731267183949Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@157ea41a: reopening flushed file at 1731267183952 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4376627a: reopening flushed file at 1731267183954 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f02b99a: reopening flushed file at 1731267183956 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 68ms, sequenceid=83, compaction requested=false at 1731267183958 (+2 ms)Writing region close event to WAL at 1731267183960 (+2 ms)Closed at 1731267183960 2024-11-10T19:33:03,961 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T19:33:03,961 INFO [M:0;8124d42bb870:41271 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T19:33:03,962 INFO [M:0;8124d42bb870:41271 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41271 2024-11-10T19:33:03,962 INFO [M:0;8124d42bb870:41271 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T19:33:03,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:03,967 INFO [RS:1;8124d42bb870:45285 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:33:03,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45285-0x101278645cc0002, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:03,967 INFO [RS:1;8124d42bb870:45285 {}] regionserver.HRegionServer(1031): Exiting; stopping=8124d42bb870,45285,1731267179468; zookeeper connection closed. 2024-11-10T19:33:03,967 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4875ebd0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4875ebd0 2024-11-10T19:33:03,968 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-10T19:33:04,092 INFO [M:0;8124d42bb870:41271 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T19:33:04,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:04,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41271-0x101278645cc0000, quorum=127.0.0.1:64354, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T19:33:04,098 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T19:33:04,100 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T19:33:04,100 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T19:33:04,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=157 (was 162), ProcessCount=11 (was 11), AvailableMemoryMB=15524 (was 15625)