2024-11-19 17:47:05,052 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 17:47:05,085 main DEBUG Took 0.029947 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-19 17:47:05,085 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-19 17:47:05,086 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-19 17:47:05,087 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-19 17:47:05,088 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,097 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-19 17:47:05,114 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,118 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,120 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,122 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,122 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,123 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,124 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,125 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,125 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,126 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,126 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,127 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,128 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,128 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,129 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,130 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,130 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,131 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,131 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-19 17:47:05,132 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,133 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-19 17:47:05,134 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-19 17:47:05,136 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-19 17:47:05,139 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-19 17:47:05,140 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-19 17:47:05,141 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-19 17:47:05,142 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-19 17:47:05,153 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-19 17:47:05,156 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-19 17:47:05,157 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-19 17:47:05,158 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-19 17:47:05,158 main DEBUG createAppenders(={Console}) 2024-11-19 17:47:05,159 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-19 17:47:05,160 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-19 17:47:05,160 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-19 17:47:05,161 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-19 17:47:05,161 main DEBUG OutputStream closed 2024-11-19 17:47:05,161 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-19 17:47:05,162 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-19 17:47:05,162 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-19 17:47:05,244 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-19 17:47:05,246 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-19 17:47:05,247 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-19 17:47:05,248 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-19 17:47:05,249 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-19 17:47:05,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-19 17:47:05,250 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-19 17:47:05,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-19 17:47:05,251 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-19 17:47:05,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-19 17:47:05,252 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-19 17:47:05,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-19 17:47:05,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-19 17:47:05,253 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-19 17:47:05,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-19 17:47:05,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-19 17:47:05,254 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-19 17:47:05,255 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-19 17:47:05,257 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19 17:47:05,258 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-19 17:47:05,258 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-19 17:47:05,259 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-19T17:47:05,277 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-19 17:47:05,281 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-19 17:47:05,281 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-19T17:47:05,588 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8 2024-11-19T17:47:05,619 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=11431 2024-11-19T17:47:05,745 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-19T17:47:05,886 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:05,908 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/cluster_b76c42cc-efcd-98df-2a35-fefd129114c6, deleteOnExit=true 2024-11-19T17:47:06,032 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/cluster_b76c42cc-efcd-98df-2a35-fefd129114c6/zookeeper_0, clientPort=60830, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/cluster_b76c42cc-efcd-98df-2a35-fefd129114c6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/cluster_b76c42cc-efcd-98df-2a35-fefd129114c6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T17:47:06,044 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60830 2024-11-19T17:47:06,044 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:06,129 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 with version=8 2024-11-19T17:47:06,130 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase-staging 2024-11-19T17:47:06,226 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-19T17:47:06,461 INFO [Time-limited test {}] client.ConnectionUtils(128): master/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:06,470 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:06,472 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:06,483 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:06,483 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:06,484 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:06,641 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T17:47:06,722 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:47:06,765 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-19T17:47:06,776 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-19T17:47:06,780 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:06,821 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 16033 (auto-detected) 2024-11-19T17:47:06,822 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-19T17:47:06,848 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39901 2024-11-19T17:47:06,877 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39901 connecting to ZooKeeper ensemble=127.0.0.1:60830 2024-11-19T17:47:06,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:399010x0, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:06,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39901-0x1015434f9490000 connected 2024-11-19T17:47:07,398 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,400 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:07,416 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5, hbase.cluster.distributed=false 2024-11-19T17:47:07,443 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:07,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39901 2024-11-19T17:47:07,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39901 2024-11-19T17:47:07,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39901 2024-11-19T17:47:07,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39901 2024-11-19T17:47:07,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39901 2024-11-19T17:47:07,543 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:07,544 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,545 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:07,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,545 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:07,548 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:47:07,551 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:07,553 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46637 2024-11-19T17:47:07,555 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46637 connecting to ZooKeeper ensemble=127.0.0.1:60830 2024-11-19T17:47:07,555 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,556 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466370x0, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:07,614 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:466370x0, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:07,614 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46637-0x1015434f9490001 connected 2024-11-19T17:47:07,624 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:47:07,638 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.cache 2024-11-19T17:47:07,646 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T17:47:07,647 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T17:47:07,649 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T17:47:07,651 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T17:47:07,651 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T17:47:07,658 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T17:47:07,661 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T17:47:07,661 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence is complete. 2024-11-19T17:47:07,664 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:47:07,667 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:47:07,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:07,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46637 2024-11-19T17:47:07,674 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46637 2024-11-19T17:47:07,676 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46637 2024-11-19T17:47:07,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46637 2024-11-19T17:47:07,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46637 2024-11-19T17:47:07,696 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:07,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,696 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,697 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:07,697 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:07,697 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:07,697 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:47:07,697 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:07,701 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40155 2024-11-19T17:47:07,703 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40155 connecting to ZooKeeper ensemble=127.0.0.1:60830 2024-11-19T17:47:07,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,704 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:07,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401550x0, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:07,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:07,754 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40155-0x1015434f9490002 connected 2024-11-19T17:47:07,755 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:47:07,756 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.cache 2024-11-19T17:47:07,756 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T17:47:07,756 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T17:47:07,756 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T17:47:07,757 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T17:47:07,757 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T17:47:07,758 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T17:47:07,758 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T17:47:07,758 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence is complete. 2024-11-19T17:47:07,759 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:47:07,760 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:47:07,763 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:07,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40155 2024-11-19T17:47:07,764 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40155 2024-11-19T17:47:07,768 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40155 2024-11-19T17:47:07,769 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40155 2024-11-19T17:47:07,772 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40155 2024-11-19T17:47:07,787 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;26d3106abcfe:39901 2024-11-19T17:47:07,788 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/26d3106abcfe,39901,1732038426294 2024-11-19T17:47:07,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,833 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/26d3106abcfe,39901,1732038426294 2024-11-19T17:47:07,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:07,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:07,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:07,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:07,892 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:07,893 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T17:47:07,894 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/26d3106abcfe,39901,1732038426294 from backup master directory 2024-11-19T17:47:07,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/26d3106abcfe,39901,1732038426294 2024-11-19T17:47:07,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:07,952 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:07,952 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=26d3106abcfe,39901,1732038426294 2024-11-19T17:47:07,956 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-19T17:47:07,959 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-19T17:47:07,967 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase.id] with ID: 9462a558-c300-4d55-b8d9-f17ae01e056b 2024-11-19T17:47:07,968 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.tmp/hbase.id 2024-11-19T17:47:07,972 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase.id] 2024-11-19T17:47:08,008 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:08,015 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T17:47:08,043 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-19T17:47:08,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,147 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:47:08,151 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T17:47:08,192 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:47:08,199 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:08,256 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store 2024-11-19T17:47:08,279 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-19T17:47:08,283 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:08,285 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:47:08,286 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:08,286 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:08,288 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:47:08,289 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:08,289 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:08,290 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038428285Disabling compacts and flushes for region at 1732038428285Disabling writes for close at 1732038428289 (+4 ms)Writing region close event to WAL at 1732038428289Closed at 1732038428289 2024-11-19T17:47:08,291 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/.initializing 2024-11-19T17:47:08,291 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,39901,1732038426294 2024-11-19T17:47:08,304 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:08,319 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26d3106abcfe%2C39901%2C1732038426294, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,39901,1732038426294, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/oldWALs, maxLogs=10 2024-11-19T17:47:08,354 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,39901,1732038426294/26d3106abcfe%2C39901%2C1732038426294.1732038428323 2024-11-19T17:47:08,355 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:08,355 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:08,356 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:08,359 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,361 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T17:47:08,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:08,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:08,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T17:47:08,440 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:08,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:08,441 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T17:47:08,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:08,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:08,447 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,452 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T17:47:08,452 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:08,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:08,453 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,456 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,457 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,460 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,460 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,463 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:08,465 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:08,471 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:08,472 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70445432, jitterRate=0.04971873760223389}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:08,480 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732038428373Initializing all the Stores at 1732038428374 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038428374Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038428375 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038428375Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038428375Cleaning up temporary data from old regions at 1732038428461 (+86 ms)Region opened successfully at 1732038428480 (+19 ms) 2024-11-19T17:47:08,481 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T17:47:08,513 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d6bb36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:08,546 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T17:47:08,557 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T17:47:08,558 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T17:47:08,561 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T17:47:08,561 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T17:47:08,567 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-19T17:47:08,567 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T17:47:08,592 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T17:47:08,601 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T17:47:08,668 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T17:47:08,672 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T17:47:08,674 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T17:47:08,742 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T17:47:08,745 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T17:47:08,751 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T17:47:08,827 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T17:47:08,830 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T17:47:08,851 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T17:47:08,877 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T17:47:08,892 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T17:47:08,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:08,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:08,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:08,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,916 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=26d3106abcfe,39901,1732038426294, sessionid=0x1015434f9490000, setting cluster-up flag (Was=false) 2024-11-19T17:47:08,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:08,977 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T17:47:08,980 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,39901,1732038426294 2024-11-19T17:47:09,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:09,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:09,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:09,034 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T17:47:09,036 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,39901,1732038426294 2024-11-19T17:47:09,041 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T17:47:09,077 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(746): ClusterId : 9462a558-c300-4d55-b8d9-f17ae01e056b 2024-11-19T17:47:09,080 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(746): ClusterId : 9462a558-c300-4d55-b8d9-f17ae01e056b 2024-11-19T17:47:09,082 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:47:09,089 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:47:09,106 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:47:09,106 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:47:09,107 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:47:09,107 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:47:09,125 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:09,131 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:47:09,132 DEBUG [RS:0;26d3106abcfe:46637 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@698d7766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:09,134 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:47:09,135 DEBUG [RS:1;26d3106abcfe:40155 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4713b565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:09,140 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T17:47:09,150 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T17:47:09,156 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;26d3106abcfe:46637 2024-11-19T17:47:09,156 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;26d3106abcfe:40155 2024-11-19T17:47:09,157 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 26d3106abcfe,39901,1732038426294 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T17:47:09,163 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:47:09,163 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:47:09,163 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:47:09,168 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:47:09,169 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:47:09,169 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:47:09,171 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,39901,1732038426294 with port=46637, startcode=1732038427508 2024-11-19T17:47:09,173 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,39901,1732038426294 with port=40155, startcode=1732038427695 2024-11-19T17:47:09,179 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:09,179 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:09,180 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:09,181 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:09,181 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/26d3106abcfe:0, corePoolSize=10, maxPoolSize=10 2024-11-19T17:47:09,181 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,181 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:09,182 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,200 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:09,201 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T17:47:09,204 DEBUG [RS:1;26d3106abcfe:40155 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:47:09,204 DEBUG [RS:0;26d3106abcfe:46637 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:47:09,208 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:09,209 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T17:47:09,220 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732038459220 2024-11-19T17:47:09,222 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T17:47:09,224 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T17:47:09,228 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T17:47:09,228 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T17:47:09,229 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T17:47:09,230 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T17:47:09,244 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,259 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T17:47:09,261 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T17:47:09,262 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T17:47:09,265 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T17:47:09,266 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T17:47:09,276 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038429268,5,FailOnTimeoutGroup] 2024-11-19T17:47:09,296 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038429277,5,FailOnTimeoutGroup] 2024-11-19T17:47:09,296 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,297 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T17:47:09,298 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,299 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,317 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T17:47:09,318 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:09,323 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47729, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:47:09,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51553, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:47:09,333 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,336 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39901 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:09,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T17:47:09,359 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:09,359 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T17:47:09,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39901 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,360 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:47:09,363 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39901 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,368 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:09,368 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T17:47:09,369 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:47:09,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:47:09,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:09,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:09,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T17:47:09,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:09,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:47:09,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:09,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:09,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T17:47:09,386 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:47:09,386 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:09,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:09,387 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T17:47:09,398 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:47:09,399 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:09,399 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:09,400 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:47:09,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:09,401 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:09,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:47:09,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:47:09,403 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:09,404 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:47:09,429 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:09,430 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60840587, jitterRate=-0.0934046059846878}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:09,433 DEBUG [RS:1;26d3106abcfe:40155 {}] zookeeper.ZKUtil(111): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,434 WARN [RS:1;26d3106abcfe:40155 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:09,434 INFO [RS:1;26d3106abcfe:40155 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:09,434 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,40155,1732038427695] 2024-11-19T17:47:09,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,46637,1732038427508] 2024-11-19T17:47:09,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732038429354Initializing all the Stores at 1732038429355 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038429355Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038429355Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038429355Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038429355Cleaning up temporary data from old regions at 1732038429402 (+47 ms)Region opened successfully at 1732038429436 (+34 ms) 2024-11-19T17:47:09,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:47:09,436 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:47:09,436 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:47:09,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:47:09,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:47:09,437 DEBUG [RS:0;26d3106abcfe:46637 {}] zookeeper.ZKUtil(111): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,437 WARN [RS:0;26d3106abcfe:46637 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:09,437 INFO [RS:0;26d3106abcfe:46637 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:09,438 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,441 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:09,442 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038429436Disabling compacts and flushes for region at 1732038429436Disabling writes for close at 1732038429437 (+1 ms)Writing region close event to WAL at 1732038429441 (+4 ms)Closed at 1732038429441 2024-11-19T17:47:09,444 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:09,444 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T17:47:09,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T17:47:09,468 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:09,472 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T17:47:09,481 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:47:09,496 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:47:09,520 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:47:09,520 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:47:09,538 INFO [RS:0;26d3106abcfe:46637 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:47:09,539 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,549 INFO [RS:1;26d3106abcfe:40155 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:47:09,549 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,553 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:47:09,556 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:47:09,559 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:47:09,559 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:47:09,561 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,561 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,562 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,562 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,563 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,564 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,564 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:09,564 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:09,564 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:09,564 DEBUG [RS:0;26d3106abcfe:46637 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:09,564 DEBUG [RS:1;26d3106abcfe:40155 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:09,573 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,573 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,574 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,574 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,574 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,574 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,40155,1732038427695-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:09,575 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,575 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,575 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,575 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,576 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,576 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,46637,1732038427508-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:09,606 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:47:09,607 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:47:09,609 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,46637,1732038427508-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,609 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,40155,1732038427695-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,609 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,609 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,609 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.Replication(171): 26d3106abcfe,40155,1732038427695 started 2024-11-19T17:47:09,609 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.Replication(171): 26d3106abcfe,46637,1732038427508 started 2024-11-19T17:47:09,623 WARN [26d3106abcfe:39901 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T17:47:09,638 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,638 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:09,638 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,46637,1732038427508, RpcServer on 26d3106abcfe/172.17.0.2:46637, sessionid=0x1015434f9490001 2024-11-19T17:47:09,638 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,40155,1732038427695, RpcServer on 26d3106abcfe/172.17.0.2:40155, sessionid=0x1015434f9490002 2024-11-19T17:47:09,639 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:47:09,639 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:47:09,639 DEBUG [RS:1;26d3106abcfe:40155 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,639 DEBUG [RS:0;26d3106abcfe:46637 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,640 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,46637,1732038427508' 2024-11-19T17:47:09,640 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,40155,1732038427695' 2024-11-19T17:47:09,640 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:47:09,640 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:47:09,641 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:47:09,641 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:47:09,642 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:47:09,642 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:47:09,642 DEBUG [RS:1;26d3106abcfe:40155 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,40155,1732038427695 2024-11-19T17:47:09,642 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,40155,1732038427695' 2024-11-19T17:47:09,642 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:47:09,642 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:47:09,642 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:47:09,642 DEBUG [RS:0;26d3106abcfe:46637 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,643 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,46637,1732038427508' 2024-11-19T17:47:09,643 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:47:09,643 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:47:09,643 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:47:09,644 DEBUG [RS:1;26d3106abcfe:40155 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:47:09,644 INFO [RS:1;26d3106abcfe:40155 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:47:09,644 INFO [RS:1;26d3106abcfe:40155 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:47:09,644 DEBUG [RS:0;26d3106abcfe:46637 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:47:09,644 INFO [RS:0;26d3106abcfe:46637 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:47:09,644 INFO [RS:0;26d3106abcfe:46637 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:47:09,749 INFO [RS:1;26d3106abcfe:40155 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:09,749 INFO [RS:0;26d3106abcfe:46637 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:09,750 INFO [RS:1;26d3106abcfe:40155 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C40155%2C1732038427695, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,40155,1732038427695, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:09,751 INFO [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C46637%2C1732038427508, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,46637,1732038427508, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:09,777 INFO [RS:1;26d3106abcfe:40155 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,40155,1732038427695/26d3106abcfe%2C40155%2C1732038427695.1732038429755 2024-11-19T17:47:09,778 DEBUG [RS:1;26d3106abcfe:40155 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:09,779 INFO [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,46637,1732038427508/26d3106abcfe%2C46637%2C1732038427508.1732038429755 2024-11-19T17:47:09,780 DEBUG [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:09,876 DEBUG [26d3106abcfe:39901 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T17:47:09,890 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(204): Hosts are {26d3106abcfe=0} racks are {/default-rack=0} 2024-11-19T17:47:09,894 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T17:47:09,894 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T17:47:09,894 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T17:47:09,894 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T17:47:09,894 INFO [26d3106abcfe:39901 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T17:47:09,894 INFO [26d3106abcfe:39901 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T17:47:09,894 DEBUG [26d3106abcfe:39901 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T17:47:09,901 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:09,908 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,46637,1732038427508, state=OPENING 2024-11-19T17:47:09,997 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T17:47:10,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:10,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:10,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:10,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,083 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,085 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:10,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,46637,1732038427508}] 2024-11-19T17:47:10,268 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:47:10,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:47:10,278 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T17:47:10,278 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:10,279 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T17:47:10,280 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C46637%2C1732038427508.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,46637,1732038427508, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:10,298 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,46637,1732038427508/26d3106abcfe%2C46637%2C1732038427508.meta.1732038430282.meta 2024-11-19T17:47:10,298 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:10,299 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:10,300 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T17:47:10,302 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T17:47:10,306 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T17:47:10,309 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T17:47:10,310 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:10,310 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T17:47:10,310 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T17:47:10,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T17:47:10,313 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:47:10,314 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:10,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:10,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T17:47:10,315 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:47:10,315 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:10,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:10,316 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T17:47:10,317 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:47:10,317 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:10,317 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:10,318 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T17:47:10,319 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:47:10,319 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:10,319 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:10,319 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:47:10,321 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:10,321 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:10,322 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:47:10,322 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:47:10,322 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:10,323 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:47:10,324 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59485786, jitterRate=-0.1135927140712738}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:10,324 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T17:47:10,326 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732038430311Writing region info on filesystem at 1732038430311Initializing all the Stores at 1732038430311Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038430312 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038430312Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038430312Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038430312Cleaning up temporary data from old regions at 1732038430322 (+10 ms)Running coprocessor post-open hooks at 1732038430324 (+2 ms)Region opened successfully at 1732038430326 (+2 ms) 2024-11-19T17:47:10,331 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732038430258 2024-11-19T17:47:10,342 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T17:47:10,343 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T17:47:10,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:10,345 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,46637,1732038427508, state=OPEN 2024-11-19T17:47:10,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:10,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:10,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:10,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,371 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:10,371 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:10,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T17:47:10,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,46637,1732038427508 in 284 msec 2024-11-19T17:47:10,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T17:47:10,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 925 msec 2024-11-19T17:47:10,383 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:10,383 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T17:47:10,402 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:47:10,403 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,46637,1732038427508, seqNum=-1] 2024-11-19T17:47:10,419 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:47:10,421 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42253, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:47:10,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3690 sec 2024-11-19T17:47:10,440 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732038430440, completionTime=-1 2024-11-19T17:47:10,443 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T17:47:10,443 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T17:47:10,462 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T17:47:10,463 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732038490462 2024-11-19T17:47:10,463 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732038550463 2024-11-19T17:47:10,463 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 20 msec 2024-11-19T17:47:10,465 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T17:47:10,471 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,471 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,471 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,473 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-26d3106abcfe:39901, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,473 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,474 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,480 DEBUG [master/26d3106abcfe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T17:47:10,500 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.547sec 2024-11-19T17:47:10,501 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T17:47:10,502 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T17:47:10,503 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T17:47:10,504 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T17:47:10,504 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T17:47:10,504 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:10,505 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T17:47:10,506 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T17:47:10,506 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T17:47:10,507 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,39901,1732038426294-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:10,587 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:10,589 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-19T17:47:10,589 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-19T17:47:10,594 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,39901,-1 for getting cluster id 2024-11-19T17:47:10,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:47:10,605 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9462a558-c300-4d55-b8d9-f17ae01e056b' 2024-11-19T17:47:10,608 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:47:10,609 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9462a558-c300-4d55-b8d9-f17ae01e056b" 2024-11-19T17:47:10,609 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46da5702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:10,609 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,39901,-1] 2024-11-19T17:47:10,613 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:47:10,615 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:10,616 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:47:10,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:10,620 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:47:10,629 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,46637,1732038427508, seqNum=-1] 2024-11-19T17:47:10,629 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:47:10,632 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40884, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:47:10,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=26d3106abcfe,39901,1732038426294 2024-11-19T17:47:10,669 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:47:10,676 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,39901,1732038426294 2024-11-19T17:47:10,690 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3dfcc359 2024-11-19T17:47:10,691 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:47:10,695 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:47:10,700 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732038430664', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:47:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 2024-11-19T17:47:10,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:47:10,714 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732038430664" procId is: 4 2024-11-19T17:47:10,714 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:10,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:47:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:10,759 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f2b5161c4e808c6538b59d6f601f2793, NAME => 'testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732038430664', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:10,774 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:10,774 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1722): Closing f2b5161c4e808c6538b59d6f601f2793, disabling compactions & flushes 2024-11-19T17:47:10,774 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:10,775 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:10,775 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. after waiting 0 ms 2024-11-19T17:47:10,775 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:10,775 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:10,775 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732038430664-pool-0 {}] regionserver.HRegion(1676): Region close journal for f2b5161c4e808c6538b59d6f601f2793: Waiting for close lock at 1732038430774Disabling compacts and flushes for region at 1732038430774Disabling writes for close at 1732038430775 (+1 ms)Writing region close event to WAL at 1732038430775Closed at 1732038430775 2024-11-19T17:47:10,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:47:10,783 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732038430777"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038430777"}]},"ts":"1732038430777"} 2024-11-19T17:47:10,789 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:47:10,790 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:47:10,794 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732038430664","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038430791"}]},"ts":"1732038430791"} 2024-11-19T17:47:10,799 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732038430664, state=ENABLING in hbase:meta 2024-11-19T17:47:10,800 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {26d3106abcfe=0} racks are {/default-rack=0} 2024-11-19T17:47:10,801 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T17:47:10,801 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T17:47:10,801 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T17:47:10,801 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T17:47:10,801 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T17:47:10,801 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T17:47:10,801 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T17:47:10,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, ASSIGN}] 2024-11-19T17:47:10,804 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, ASSIGN 2024-11-19T17:47:10,805 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, ASSIGN; state=OFFLINE, location=26d3106abcfe,46637,1732038427508; forceNewPlan=false, retain=false 2024-11-19T17:47:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:10,959 INFO [26d3106abcfe:39901 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T17:47:10,960 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=OPENING, regionLocation=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:10,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, ASSIGN because future has completed 2024-11-19T17:47:10,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508}] 2024-11-19T17:47:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:11,123 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:11,124 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f2b5161c4e808c6538b59d6f601f2793, NAME => 'testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:11,124 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732038430664 f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,124 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:11,125 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,125 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,126 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,131 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2b5161c4e808c6538b59d6f601f2793 columnFamilyName family 2024-11-19T17:47:11,131 DEBUG [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:11,131 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(327): Store=f2b5161c4e808c6538b59d6f601f2793/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:11,132 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,133 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,133 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,133 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,133 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,134 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,139 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:11,140 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f2b5161c4e808c6538b59d6f601f2793; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60266500, jitterRate=-0.10195916891098022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T17:47:11,140 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:11,141 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f2b5161c4e808c6538b59d6f601f2793: Running coprocessor pre-open hook at 1732038431125Writing region info on filesystem at 1732038431125Initializing all the Stores at 1732038431125Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038431125Cleaning up temporary data from old regions at 1732038431133 (+8 ms)Running coprocessor post-open hooks at 1732038431140 (+7 ms)Region opened successfully at 1732038431141 (+1 ms) 2024-11-19T17:47:11,142 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793., pid=6, masterSystemTime=1732038431119 2024-11-19T17:47:11,146 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:11,146 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:11,146 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:11,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508 because future has completed 2024-11-19T17:47:11,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T17:47:11,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508 in 187 msec 2024-11-19T17:47:11,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T17:47:11,159 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, ASSIGN in 354 msec 2024-11-19T17:47:11,160 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:47:11,160 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732038430664","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038431160"}]},"ts":"1732038431160"} 2024-11-19T17:47:11,163 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732038430664, state=ENABLED in hbase:meta 2024-11-19T17:47:11,165 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:47:11,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732038430664 in 463 msec 2024-11-19T17:47:11,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:11,357 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732038430664 get assigned. Timeout = 60000ms 2024-11-19T17:47:11,357 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732038430664 completed 2024-11-19T17:47:11,359 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T17:47:11,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732038430664 assigned to meta. Checking AM states. 2024-11-19T17:47:11,367 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T17:47:11,367 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732038430664 assigned. 2024-11-19T17:47:11,376 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732038430664', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793., hostname=26d3106abcfe,46637,1732038427508, seqNum=2] 2024-11-19T17:47:11,388 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2b5161c4e808c6538b59d6f601f2793 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T17:47:11,425 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:11,471 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/.tmp/family/f60d15ae1e924158885cf98b808daf28 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T17:47:11,476 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/.tmp/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:11,513 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/.tmp/family/f60d15ae1e924158885cf98b808daf28 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:11,515 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, delay=959 ms 2024-11-19T17:47:11,516 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T17:47:11,523 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f2b5161c4e808c6538b59d6f601f2793 in 134ms, sequenceid=6, compaction requested=false 2024-11-19T17:47:11,524 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732038430664' 2024-11-19T17:47:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f2b5161c4e808c6538b59d6f601f2793: 2024-11-19T17:47:12,479 DEBUG [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f60d15ae1e924158885cf98b808daf28_0. 2024-11-19T17:47:12,486 DEBUG [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f60d15ae1e924158885cf98b808daf28_117. 2024-11-19T17:47:12,486 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, with total blocks 2, and data blocks 1 2024-11-19T17:47:12,486 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T17:47:12,487 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T17:47:12,487 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28. Will sleep 100ms and try the verification again. 2024-11-19T17:47:12,543 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f2b5161c4e808c6538b59d6f601f2793, source=26d3106abcfe,46637,1732038427508, destination=26d3106abcfe,40155,1732038427695, warming up region on 26d3106abcfe,40155,1732038427695 2024-11-19T17:47:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:47:12,545 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f2b5161c4e808c6538b59d6f601f2793, source=26d3106abcfe,46637,1732038427508, destination=26d3106abcfe,40155,1732038427695, running balancer 2024-11-19T17:47:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE 2024-11-19T17:47:12,547 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE 2024-11-19T17:47:12,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:47:12,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,548 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=CLOSING, regionLocation=26d3106abcfe,46637,1732038427508 2024-11-19T17:47:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(7855): Warmup {ENCODED => f2b5161c4e808c6538b59d6f601f2793, NAME => 'testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:12,549 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:12,550 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2b5161c4e808c6538b59d6f601f2793 columnFamilyName family 2024-11-19T17:47:12,550 DEBUG [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:12,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE because future has completed 2024-11-19T17:47:12,552 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-19T17:47:12,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508}] 2024-11-19T17:47:12,555 DEBUG [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:12,555 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(327): Store=f2b5161c4e808c6538b59d6f601f2793/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1722): Closing f2b5161c4e808c6538b59d6f601f2793, disabling compactions & flushes 2024-11-19T17:47:12,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. after waiting 0 ms 2024-11-19T17:47:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,556 DEBUG [StoreFileCloser-f2b5161c4e808c6538b59d6f601f2793-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:12,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40155 {}] regionserver.HRegion(1676): Region close journal for f2b5161c4e808c6538b59d6f601f2793: Waiting for close lock at 1732038432555Disabling compacts and flushes for region at 1732038432555Disabling writes for close at 1732038432555Writing region close event to WAL at 1732038432557 (+2 ms)Closed at 1732038432557 2024-11-19T17:47:12,559 WARN [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-19T17:47:12,560 DEBUG [hfile-prefetch-1732038431515 {}] hfile.PrefetchExecutor(121): Prefetch completed for f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39901 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-19T17:47:12,658 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T17:47:12,658 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T17:47:12,659 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:12,659 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:12,665 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T17:47:12,665 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T17:47:12,666 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:12,666 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:12,680 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:12,680 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038432666 2024-11-19T17:47:12,680 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T17:47:12,684 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-19T17:47:12,684 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732038432658 2024-11-19T17:47:12,684 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T17:47:12,710 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:12,710 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-19T17:47:12,711 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f2b5161c4e808c6538b59d6f601f2793, disabling compactions & flushes 2024-11-19T17:47:12,711 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,711 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,711 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. after waiting 0 ms 2024-11-19T17:47:12,711 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,713 DEBUG [StoreFileCloser-f2b5161c4e808c6538b59d6f601f2793-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file f60d15ae1e924158885cf98b808daf28, starting offset: 0, end offset: 9223372036854775807 2024-11-19T17:47:12,714 DEBUG [StoreFileCloser-f2b5161c4e808c6538b59d6f601f2793-family-1 {}] bucket.BucketCache(856): removed key f60d15ae1e924158885cf98b808daf28_0 from back map with offset lock 2094080 in the evict process 2024-11-19T17:47:12,720 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T17:47:12,720 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:12,721 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f2b5161c4e808c6538b59d6f601f2793: Waiting for close lock at 1732038432711Running coprocessor pre-close hooks at 1732038432711Disabling compacts and flushes for region at 1732038432711Disabling writes for close at 1732038432711Writing region close event to WAL at 1732038432715 (+4 ms)Running coprocessor post-close hooks at 1732038432720 (+5 ms)Closed at 1732038432720 2024-11-19T17:47:12,721 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f2b5161c4e808c6538b59d6f601f2793 move to 26d3106abcfe,40155,1732038427695 record at close sequenceid=6 2024-11-19T17:47:12,725 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:12,726 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=CLOSED 2024-11-19T17:47:12,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508 because future has completed 2024-11-19T17:47:12,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T17:47:12,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,46637,1732038427508 in 179 msec 2024-11-19T17:47:12,735 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE; state=CLOSED, location=26d3106abcfe,40155,1732038427695; forceNewPlan=false, retain=false 2024-11-19T17:47:12,780 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T17:47:12,780 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T17:47:12,780 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:12,781 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:12,784 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:12,784 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038432780 2024-11-19T17:47:12,784 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T17:47:12,885 INFO [26d3106abcfe:39901 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T17:47:12,886 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=OPENING, regionLocation=26d3106abcfe,40155,1732038427695 2024-11-19T17:47:12,893 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE because future has completed 2024-11-19T17:47:12,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,40155,1732038427695}] 2024-11-19T17:47:13,052 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,053 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f2b5161c4e808c6538b59d6f601f2793, NAME => 'testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:13,054 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732038430664 f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,054 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:13,054 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,054 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,056 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,057 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2b5161c4e808c6538b59d6f601f2793 columnFamilyName family 2024-11-19T17:47:13,057 DEBUG [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:13,060 DEBUG [StoreFileOpener-f2b5161c4e808c6538b59d6f601f2793-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28, delay=993 ms 2024-11-19T17:47:13,062 DEBUG [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:13,062 INFO [StoreOpener-f2b5161c4e808c6538b59d6f601f2793-1 {}] regionserver.HStore(327): Store=f2b5161c4e808c6538b59d6f601f2793/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:13,062 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,064 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,064 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,064 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,065 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,070 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,070 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f2b5161c4e808c6538b59d6f601f2793; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69731561, jitterRate=0.039081230759620667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T17:47:13,071 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,072 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f2b5161c4e808c6538b59d6f601f2793: Running coprocessor pre-open hook at 1732038433055Writing region info on filesystem at 1732038433055Initializing all the Stores at 1732038433055Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038433055Cleaning up temporary data from old regions at 1732038433065 (+10 ms)Running coprocessor post-open hooks at 1732038433071 (+6 ms)Region opened successfully at 1732038433071 2024-11-19T17:47:13,072 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793., pid=9, masterSystemTime=1732038433048 2024-11-19T17:47:13,075 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,076 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,076 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f2b5161c4e808c6538b59d6f601f2793, regionState=OPEN, openSeqNum=10, regionLocation=26d3106abcfe,40155,1732038427695 2024-11-19T17:47:13,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,40155,1732038427695 because future has completed 2024-11-19T17:47:13,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-19T17:47:13,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f2b5161c4e808c6538b59d6f601f2793, server=26d3106abcfe,40155,1732038427695 in 186 msec 2024-11-19T17:47:13,084 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732038430664, region=f2b5161c4e808c6538b59d6f601f2793, REOPEN/MOVE in 537 msec 2024-11-19T17:47:13,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T17:47:13,100 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:47:13,100 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:13,103 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,104 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:47:13,104 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T17:47:13,104 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-19T17:47:13,105 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=26d3106abcfe,39901,1732038426294 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:13,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:13,124 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:47:13,125 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:47:13,125 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:13,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,126 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,46637,1732038427508' ***** 2024-11-19T17:47:13,126 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:47:13,126 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,40155,1732038427695' ***** 2024-11-19T17:47:13,126 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:47:13,126 INFO [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T17:47:13,126 INFO [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T17:47:13,127 INFO [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/bucket.persistence 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T17:47:13,127 INFO [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T17:47:13,128 INFO [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T17:47:13,128 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T17:47:13,128 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T17:47:13,138 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1487): Thread RS:0;26d3106abcfe:46637 started persisting bucket cache to file 2024-11-19T17:47:13,138 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1487): Thread RS:1;26d3106abcfe:40155 started persisting bucket cache to file 2024-11-19T17:47:13,138 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:13,138 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:13,138 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:13,138 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:13,140 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:13,140 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:13,141 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038433138 2024-11-19T17:47:13,141 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038433138 2024-11-19T17:47:13,141 DEBUG [RS:1;26d3106abcfe:40155 {}] bucket.BucketCache(1509): Thread RS:1;26d3106abcfe:40155 finished persisting bucket cache to file, renaming 2024-11-19T17:47:13,141 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:47:13,141 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:47:13,141 INFO [RS:1;26d3106abcfe:40155 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:47:13,141 INFO [RS:1;26d3106abcfe:40155 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:47:13,141 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(3091): Received CLOSE for f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,142 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,40155,1732038427695 2024-11-19T17:47:13,142 INFO [RS:1;26d3106abcfe:40155 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:13,142 INFO [RS:1;26d3106abcfe:40155 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;26d3106abcfe:40155. 2024-11-19T17:47:13,142 DEBUG [RS:1;26d3106abcfe:40155 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:13,142 DEBUG [RS:1;26d3106abcfe:40155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,142 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:13,143 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T17:47:13,143 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1325): Online Regions={f2b5161c4e808c6538b59d6f601f2793=testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793.} 2024-11-19T17:47:13,143 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f2b5161c4e808c6538b59d6f601f2793, disabling compactions & flushes 2024-11-19T17:47:13,143 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,143 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,143 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. after waiting 0 ms 2024-11-19T17:47:13,143 DEBUG [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1351): Waiting on f2b5161c4e808c6538b59d6f601f2793 2024-11-19T17:47:13,143 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,144 DEBUG [StoreFileCloser-f2b5161c4e808c6538b59d6f601f2793-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/family/f60d15ae1e924158885cf98b808daf28 2024-11-19T17:47:13,148 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnRegionMove1732038430664/f2b5161c4e808c6538b59d6f601f2793/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T17:47:13,148 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,148 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f2b5161c4e808c6538b59d6f601f2793: Waiting for close lock at 1732038433142Running coprocessor pre-close hooks at 1732038433143 (+1 ms)Disabling compacts and flushes for region at 1732038433143Disabling writes for close at 1732038433143Writing region close event to WAL at 1732038433144 (+1 ms)Running coprocessor post-close hooks at 1732038433148 (+4 ms)Closed at 1732038433148 2024-11-19T17:47:13,148 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793. 2024-11-19T17:47:13,177 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:47:13,190 DEBUG [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1509): Thread RS:0;26d3106abcfe:46637 finished persisting bucket cache to file, renaming 2024-11-19T17:47:13,191 WARN [RS:0;26d3106abcfe:46637 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-19T17:47:13,191 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:13,191 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:13,191 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:47:13,191 INFO [RS:0;26d3106abcfe:46637 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:47:13,191 INFO [RS:0;26d3106abcfe:46637 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:47:13,191 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,46637,1732038427508 2024-11-19T17:47:13,192 INFO [RS:0;26d3106abcfe:46637 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:13,192 INFO [RS:0;26d3106abcfe:46637 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;26d3106abcfe:46637. 2024-11-19T17:47:13,192 DEBUG [RS:0;26d3106abcfe:46637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:13,192 DEBUG [RS:0;26d3106abcfe:46637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,193 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:47:13,193 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:47:13,193 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:47:13,193 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T17:47:13,194 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-19T17:47:13,194 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-19T17:47:13,194 DEBUG [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-19T17:47:13,194 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:13,195 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:47:13,195 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:47:13,195 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:47:13,195 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:47:13,195 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:47:13,196 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-19T17:47:13,209 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6e8350a44d4041349c03806239740d4c 2024-11-19T17:47:13,228 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/6e8350a44d4041349c03806239740d4c is 205, key is testBlockEvictionOnRegionMove1732038430664,,1732038430696.f2b5161c4e808c6538b59d6f601f2793./info:regioninfo/1732038433076/Put/seqid=0 2024-11-19T17:47:13,229 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/6e8350a44d4041349c03806239740d4c 2024-11-19T17:47:13,236 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 841db111b7434432b9e3b0eb4b8ff2dc 2024-11-19T17:47:13,248 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/841db111b7434432b9e3b0eb4b8ff2dc is 43, key is default/ns:d/1732038430425/Put/seqid=0 2024-11-19T17:47:13,248 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/841db111b7434432b9e3b0eb4b8ff2dc 2024-11-19T17:47:13,256 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a1ffb878729e4528865d1372167284bc 2024-11-19T17:47:13,268 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/a1ffb878729e4528865d1372167284bc is 78, key is testBlockEvictionOnRegionMove1732038430664/table:state/1732038431160/Put/seqid=0 2024-11-19T17:47:13,268 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/a1ffb878729e4528865d1372167284bc 2024-11-19T17:47:13,271 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/6e8350a44d4041349c03806239740d4c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/6e8350a44d4041349c03806239740d4c 2024-11-19T17:47:13,272 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/6e8350a44d4041349c03806239740d4c, delay=970 ms 2024-11-19T17:47:13,273 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/6e8350a44d4041349c03806239740d4c, entries=15, sequenceid=15, filesize=7.6 K 2024-11-19T17:47:13,273 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/841db111b7434432b9e3b0eb4b8ff2dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/841db111b7434432b9e3b0eb4b8ff2dc 2024-11-19T17:47:13,274 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/841db111b7434432b9e3b0eb4b8ff2dc, delay=938 ms 2024-11-19T17:47:13,275 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/841db111b7434432b9e3b0eb4b8ff2dc, entries=2, sequenceid=15, filesize=5.0 K 2024-11-19T17:47:13,276 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/a1ffb878729e4528865d1372167284bc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/a1ffb878729e4528865d1372167284bc 2024-11-19T17:47:13,277 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/a1ffb878729e4528865d1372167284bc, delay=924 ms 2024-11-19T17:47:13,278 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/a1ffb878729e4528865d1372167284bc, entries=2, sequenceid=15, filesize=5.3 K 2024-11-19T17:47:13,279 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false 2024-11-19T17:47:13,279 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-19T17:47:13,280 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/6e8350a44d4041349c03806239740d4c 2024-11-19T17:47:13,281 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/841db111b7434432b9e3b0eb4b8ff2dc 2024-11-19T17:47:13,281 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/a1ffb878729e4528865d1372167284bc 2024-11-19T17:47:13,285 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-19T17:47:13,286 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:47:13,286 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:13,286 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038433194Running coprocessor pre-close hooks at 1732038433195 (+1 ms)Disabling compacts and flushes for region at 1732038433195Disabling writes for close at 1732038433195Obtaining lock to block concurrent updates at 1732038433196 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732038433196Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732038433197 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732038433198 (+1 ms)Flushing 1588230740/info: creating writer at 1732038433198Flushing 1588230740/info: appending metadata at 1732038433227 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732038433227Flushing 1588230740/ns: creating writer at 1732038433231 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732038433247 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732038433247Flushing 1588230740/table: creating writer at 1732038433250 (+3 ms)Flushing 1588230740/table: appending metadata at 1732038433267 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732038433267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ab6bc50: reopening flushed file at 1732038433270 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29925d27: reopening flushed file at 1732038433273 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d9b1fb0: reopening flushed file at 1732038433276 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false at 1732038433279 (+3 ms)Writing region close event to WAL at 1732038433282 (+3 ms)Running coprocessor post-close hooks at 1732038433285 (+3 ms)Closed at 1732038433286 (+1 ms) 2024-11-19T17:47:13,286 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:13,343 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,40155,1732038427695; all regions closed. 2024-11-19T17:47:13,349 DEBUG [RS:1;26d3106abcfe:40155 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:13,349 INFO [RS:1;26d3106abcfe:40155 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C40155%2C1732038427695:(num 1732038429755) 2024-11-19T17:47:13,349 DEBUG [RS:1;26d3106abcfe:40155 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,349 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:13,349 INFO [RS:1;26d3106abcfe:40155 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:47:13,350 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:13,350 INFO [RS:1;26d3106abcfe:40155 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40155 2024-11-19T17:47:13,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,40155,1732038427695 2024-11-19T17:47:13,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:13,370 INFO [RS:1;26d3106abcfe:40155 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:13,372 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,40155,1732038427695] 2024-11-19T17:47:13,391 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,40155,1732038427695 already deleted, retry=false 2024-11-19T17:47:13,391 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26d3106abcfe,40155,1732038427695 expired; onlineServers=1 2024-11-19T17:47:13,395 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,46637,1732038427508; all regions closed. 2024-11-19T17:47:13,397 DEBUG [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:13,397 INFO [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C46637%2C1732038427508.meta:.meta(num 1732038430282) 2024-11-19T17:47:13,399 DEBUG [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:13,400 INFO [RS:0;26d3106abcfe:46637 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C46637%2C1732038427508:(num 1732038429755) 2024-11-19T17:47:13,400 DEBUG [RS:0;26d3106abcfe:46637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:13,400 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:13,400 INFO [RS:0;26d3106abcfe:46637 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:13,400 INFO [RS:0;26d3106abcfe:46637 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:13,400 INFO [RS:0;26d3106abcfe:46637 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:13,400 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:13,401 INFO [RS:0;26d3106abcfe:46637 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46637 2024-11-19T17:47:13,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,46637,1732038427508 2024-11-19T17:47:13,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:13,412 INFO [RS:0;26d3106abcfe:46637 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:13,423 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,46637,1732038427508] 2024-11-19T17:47:13,433 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,46637,1732038427508 already deleted, retry=false 2024-11-19T17:47:13,433 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26d3106abcfe,46637,1732038427508 expired; onlineServers=0 2024-11-19T17:47:13,434 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '26d3106abcfe,39901,1732038426294' ***** 2024-11-19T17:47:13,434 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T17:47:13,434 INFO [M:0;26d3106abcfe:39901 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:13,434 INFO [M:0;26d3106abcfe:39901 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:13,434 DEBUG [M:0;26d3106abcfe:39901 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T17:47:13,434 DEBUG [M:0;26d3106abcfe:39901 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T17:47:13,434 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038429268 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038429268,5,FailOnTimeoutGroup] 2024-11-19T17:47:13,435 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038429277 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038429277,5,FailOnTimeoutGroup] 2024-11-19T17:47:13,435 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T17:47:13,435 INFO [M:0;26d3106abcfe:39901 {}] hbase.ChoreService(370): Chore service for: master/26d3106abcfe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:13,435 INFO [M:0;26d3106abcfe:39901 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:13,435 DEBUG [M:0;26d3106abcfe:39901 {}] master.HMaster(1795): Stopping service threads 2024-11-19T17:47:13,435 INFO [M:0;26d3106abcfe:39901 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T17:47:13,435 INFO [M:0;26d3106abcfe:39901 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:47:13,436 INFO [M:0;26d3106abcfe:39901 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T17:47:13,437 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T17:47:13,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:13,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:13,444 DEBUG [M:0;26d3106abcfe:39901 {}] zookeeper.ZKUtil(347): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T17:47:13,444 WARN [M:0;26d3106abcfe:39901 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T17:47:13,444 INFO [M:0;26d3106abcfe:39901 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.lastflushedseqids 2024-11-19T17:47:13,459 INFO [M:0;26d3106abcfe:39901 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T17:47:13,459 INFO [M:0;26d3106abcfe:39901 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T17:47:13,459 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:47:13,459 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:13,459 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:13,460 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-19T17:47:13,460 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:13,460 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-19T17:47:13,472 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4221e2df7b54473d892ee506d1c26bcf 2024-11-19T17:47:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40155-0x1015434f9490002, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,482 INFO [RS:1;26d3106abcfe:40155 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:13,482 INFO [RS:1;26d3106abcfe:40155 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,40155,1732038427695; zookeeper connection closed. 2024-11-19T17:47:13,483 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@b4b813d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@b4b813d 2024-11-19T17:47:13,490 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4221e2df7b54473d892ee506d1c26bcf is 82, key is hbase:meta,,1/info:regioninfo/1732038430343/Put/seqid=0 2024-11-19T17:47:13,491 INFO [M:0;26d3106abcfe:39901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4221e2df7b54473d892ee506d1c26bcf 2024-11-19T17:47:13,501 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57a9dc91e3e74727bbbd117233d3d898 2024-11-19T17:47:13,517 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57a9dc91e3e74727bbbd117233d3d898 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732038431167/Put/seqid=0 2024-11-19T17:47:13,518 INFO [M:0;26d3106abcfe:39901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57a9dc91e3e74727bbbd117233d3d898 2024-11-19T17:47:13,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,523 INFO [RS:0;26d3106abcfe:46637 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:13,523 INFO [RS:0;26d3106abcfe:46637 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,46637,1732038427508; zookeeper connection closed. 2024-11-19T17:47:13,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46637-0x1015434f9490001, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,523 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40c688fd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40c688fd 2024-11-19T17:47:13,524 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-19T17:47:13,530 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07f40a79597f41009a130ec8ff7effe2 2024-11-19T17:47:13,548 DEBUG [M:0;26d3106abcfe:39901 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07f40a79597f41009a130ec8ff7effe2 is 69, key is 26d3106abcfe,40155,1732038427695/rs:state/1732038429341/Put/seqid=0 2024-11-19T17:47:13,548 INFO [M:0;26d3106abcfe:39901 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07f40a79597f41009a130ec8ff7effe2 2024-11-19T17:47:13,550 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4221e2df7b54473d892ee506d1c26bcf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4221e2df7b54473d892ee506d1c26bcf 2024-11-19T17:47:13,552 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4221e2df7b54473d892ee506d1c26bcf, entries=8, sequenceid=84, filesize=5.5 K 2024-11-19T17:47:13,552 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/57a9dc91e3e74727bbbd117233d3d898 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57a9dc91e3e74727bbbd117233d3d898 2024-11-19T17:47:13,554 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/57a9dc91e3e74727bbbd117233d3d898, entries=9, sequenceid=84, filesize=6.5 K 2024-11-19T17:47:13,554 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07f40a79597f41009a130ec8ff7effe2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07f40a79597f41009a130ec8ff7effe2 2024-11-19T17:47:13,555 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07f40a79597f41009a130ec8ff7effe2, entries=2, sequenceid=84, filesize=5.1 K 2024-11-19T17:47:13,556 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false 2024-11-19T17:47:13,558 INFO [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:13,558 DEBUG [M:0;26d3106abcfe:39901 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038433459Disabling compacts and flushes for region at 1732038433459Disabling writes for close at 1732038433460 (+1 ms)Obtaining lock to block concurrent updates at 1732038433460Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732038433460Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732038433460Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732038433461 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732038433461Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732038433489 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732038433489Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732038433494 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732038433516 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732038433516Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732038433520 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732038433547 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732038433547Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37791a09: reopening flushed file at 1732038433550 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51161dc7: reopening flushed file at 1732038433552 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48d684ca: reopening flushed file at 1732038433554 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 96ms, sequenceid=84, compaction requested=false at 1732038433556 (+2 ms)Writing region close event to WAL at 1732038433558 (+2 ms)Closed at 1732038433558 2024-11-19T17:47:13,560 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:13,560 INFO [M:0;26d3106abcfe:39901 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T17:47:13,560 INFO [M:0;26d3106abcfe:39901 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39901 2024-11-19T17:47:13,560 INFO [M:0;26d3106abcfe:39901 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:13,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,670 INFO [M:0;26d3106abcfe:39901 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:13,670 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39901-0x1015434f9490000, quorum=127.0.0.1:60830, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:13,675 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T17:47:13,700 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T17:47:13,700 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,709 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/26d3106abcfe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/26d3106abcfe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732038433272 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/26d3106abcfe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732038431515 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/26d3106abcfe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732038433274 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732038433060 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=557 (was 562), ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=10691 (was 11431) 2024-11-19T17:47:13,716 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=557, ProcessCount=11, AvailableMemoryMB=10690 2024-11-19T17:47:13,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/hadoop-log-dir so I do NOT create it in target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9 2024-11-19T17:47:13,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/hadoop-tmp-dir so I do NOT create it in target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9 2024-11-19T17:47:13,717 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,720 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/cluster_80396b2c-9ddf-05e4-61b7-75ad6ccbf987, deleteOnExit=true 2024-11-19T17:47:13,723 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/cluster_80396b2c-9ddf-05e4-61b7-75ad6ccbf987/zookeeper_0, clientPort=64906, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/cluster_80396b2c-9ddf-05e4-61b7-75ad6ccbf987/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/cluster_80396b2c-9ddf-05e4-61b7-75ad6ccbf987/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-19T17:47:13,725 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64906 2024-11-19T17:47:13,735 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 with version=8 2024-11-19T17:47:13,735 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase-staging 2024-11-19T17:47:13,738 INFO [Time-limited test {}] client.ConnectionUtils(128): master/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:13,738 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-19T17:47:13,739 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:13,740 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44853 2024-11-19T17:47:13,742 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44853 connecting to ZooKeeper ensemble=127.0.0.1:64906 2024-11-19T17:47:13,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:448530x0, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:13,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44853-0x101543517660000 connected 2024-11-19T17:47:13,884 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,884 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,886 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5, hbase.cluster.distributed=false 2024-11-19T17:47:13,888 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:13,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44853 2024-11-19T17:47:13,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44853 2024-11-19T17:47:13,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44853 2024-11-19T17:47:13,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44853 2024-11-19T17:47:13,889 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44853 2024-11-19T17:47:13,904 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:13,904 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:47:13,905 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:13,905 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37277 2024-11-19T17:47:13,907 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37277 connecting to ZooKeeper ensemble=127.0.0.1:64906 2024-11-19T17:47:13,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:372770x0, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:13,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37277-0x101543517660001 connected 2024-11-19T17:47:13,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,921 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:47:13,922 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache 2024-11-19T17:47:13,922 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T17:47:13,922 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T17:47:13,922 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T17:47:13,923 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T17:47:13,923 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T17:47:13,923 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T17:47:13,923 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence is complete. 2024-11-19T17:47:13,924 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T17:47:13,926 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:47:13,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:47:13,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37277 2024-11-19T17:47:13,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37277 2024-11-19T17:47:13,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37277 2024-11-19T17:47:13,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37277 2024-11-19T17:47:13,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37277 2024-11-19T17:47:13,951 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:47:13,951 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:13,952 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41933 2024-11-19T17:47:13,954 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41933 connecting to ZooKeeper ensemble=127.0.0.1:64906 2024-11-19T17:47:13,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,954 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:13,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:419330x0, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:13,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:419330x0, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:13,968 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41933-0x101543517660002 connected 2024-11-19T17:47:13,968 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:47:13,969 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache 2024-11-19T17:47:13,969 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T17:47:13,969 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T17:47:13,970 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T17:47:13,970 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T17:47:13,970 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-19T17:47:13,970 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T17:47:13,970 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence is complete. 2024-11-19T17:47:13,972 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T17:47:13,973 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:47:13,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-19T17:47:13,976 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:13,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41933 2024-11-19T17:47:13,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41933 2024-11-19T17:47:13,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41933 2024-11-19T17:47:13,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41933 2024-11-19T17:47:13,984 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41933 2024-11-19T17:47:13,996 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;26d3106abcfe:44853 2024-11-19T17:47:13,997 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,011 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:14,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,028 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T17:47:14,029 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/26d3106abcfe,44853,1732038433738 from backup master directory 2024-11-19T17:47:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,041 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:14,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-19T17:47:14,041 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,043 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase.id] with ID: b8de8b75-c499-49a7-86c7-aaeb7f384aa7 2024-11-19T17:47:14,043 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.tmp/hbase.id 2024-11-19T17:47:14,048 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/hbase.id] 2024-11-19T17:47:14,086 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:14,086 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-19T17:47:14,089 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-19T17:47:14,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,105 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:47:14,106 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-19T17:47:14,106 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:14,144 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:47:14,158 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:14,158 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:14,158 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038434158Disabling compacts and flushes for region at 1732038434158Disabling writes for close at 1732038434158Writing region close event to WAL at 1732038434158Closed at 1732038434158 2024-11-19T17:47:14,159 WARN [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/.initializing 2024-11-19T17:47:14,159 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,166 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:14,171 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=26d3106abcfe%2C44853%2C1732038433738, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,44853,1732038433738, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/oldWALs, maxLogs=10 2024-11-19T17:47:14,194 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/WALs/26d3106abcfe,44853,1732038433738/26d3106abcfe%2C44853%2C1732038433738.1732038434171 2024-11-19T17:47:14,195 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:14,195 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:14,195 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:14,195 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,195 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,196 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-19T17:47:14,202 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:14,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-19T17:47:14,208 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:14,208 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,212 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-19T17:47:14,212 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:14,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-19T17:47:14,217 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,217 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:14,218 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,218 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,218 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,219 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,219 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,219 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:14,219 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-19T17:47:14,226 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:14,226 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61084714, jitterRate=-0.08976683020591736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:14,228 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732038434196Initializing all the Stores at 1732038434196Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038434196Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038434196Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038434196Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038434196Cleaning up temporary data from old regions at 1732038434219 (+23 ms)Region opened successfully at 1732038434227 (+8 ms) 2024-11-19T17:47:14,228 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-19T17:47:14,232 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e535b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:14,233 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-19T17:47:14,234 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-19T17:47:14,234 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-19T17:47:14,234 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-19T17:47:14,234 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-19T17:47:14,235 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-19T17:47:14,235 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-19T17:47:14,237 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-19T17:47:14,239 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-19T17:47:14,251 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-19T17:47:14,252 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-19T17:47:14,253 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-19T17:47:14,262 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-19T17:47:14,262 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-19T17:47:14,264 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-19T17:47:14,272 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-19T17:47:14,274 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-19T17:47:14,283 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-19T17:47:14,288 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-19T17:47:14,299 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-19T17:47:14,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:14,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:14,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:14,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,310 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=26d3106abcfe,44853,1732038433738, sessionid=0x101543517660000, setting cluster-up flag (Was=false) 2024-11-19T17:47:14,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,367 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-19T17:47:14,369 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,420 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-19T17:47:14,421 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=26d3106abcfe,44853,1732038433738 2024-11-19T17:47:14,422 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-19T17:47:14,425 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:14,425 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-19T17:47:14,426 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-19T17:47:14,426 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 26d3106abcfe,44853,1732038433738 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-19T17:47:14,427 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:14,427 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/26d3106abcfe:0, corePoolSize=5, maxPoolSize=5 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/26d3106abcfe:0, corePoolSize=10, maxPoolSize=10 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:14,428 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,433 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:14,434 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-19T17:47:14,434 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,435 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-19T17:47:14,448 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732038464448 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-19T17:47:14,449 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,450 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-19T17:47:14,450 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-19T17:47:14,450 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-19T17:47:14,450 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-19T17:47:14,450 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-19T17:47:14,450 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038434450,5,FailOnTimeoutGroup] 2024-11-19T17:47:14,451 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038434451,5,FailOnTimeoutGroup] 2024-11-19T17:47:14,451 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,452 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-19T17:47:14,452 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,452 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,469 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-19T17:47:14,469 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:14,475 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:14,476 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T17:47:14,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:47:14,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:14,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T17:47:14,484 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:47:14,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:14,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T17:47:14,487 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(746): ClusterId : b8de8b75-c499-49a7-86c7-aaeb7f384aa7 2024-11-19T17:47:14,487 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(746): ClusterId : b8de8b75-c499-49a7-86c7-aaeb7f384aa7 2024-11-19T17:47:14,487 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:47:14,487 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:47:14,489 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:47:14,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:14,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T17:47:14,500 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:47:14,500 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:47:14,500 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:47:14,500 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:47:14,504 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:47:14,504 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:14,505 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:14,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:47:14,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:14,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:14,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:47:14,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:47:14,506 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:14,507 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:47:14,510 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:47:14,510 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:47:14,511 DEBUG [RS:0;26d3106abcfe:37277 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff3d236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:14,511 DEBUG [RS:1;26d3106abcfe:41933 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eed3e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:14,525 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:14,525 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75011313, jitterRate=0.11775566637516022}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732038434475Initializing all the Stores at 1732038434476 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038434476Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038434476Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038434476Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038434476Cleaning up temporary data from old regions at 1732038434506 (+30 ms)Region opened successfully at 1732038434526 (+20 ms) 2024-11-19T17:47:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:47:14,526 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:47:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:47:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:47:14,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:47:14,527 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;26d3106abcfe:37277 2024-11-19T17:47:14,527 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:47:14,527 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:47:14,527 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:47:14,527 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:14,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038434526Disabling compacts and flushes for region at 1732038434526Disabling writes for close at 1732038434526Writing region close event to WAL at 1732038434527 (+1 ms)Closed at 1732038434527 2024-11-19T17:47:14,528 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,44853,1732038433738 with port=37277, startcode=1732038433903 2024-11-19T17:47:14,528 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:14,528 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-19T17:47:14,528 DEBUG [RS:0;26d3106abcfe:37277 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:47:14,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T17:47:14,529 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:14,530 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-19T17:47:14,536 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;26d3106abcfe:41933 2024-11-19T17:47:14,536 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:47:14,536 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:47:14,536 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:47:14,538 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,44853,1732038433738 with port=41933, startcode=1732038433950 2024-11-19T17:47:14,539 DEBUG [RS:1;26d3106abcfe:41933 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:47:14,539 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:47:14,540 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,540 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,542 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:14,542 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T17:47:14,542 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:47:14,542 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50789, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:47:14,543 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,543 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,545 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:14,545 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T17:47:14,545 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:47:14,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:14,584 DEBUG [RS:0;26d3106abcfe:37277 {}] zookeeper.ZKUtil(111): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,584 WARN [RS:0;26d3106abcfe:37277 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:14,584 INFO [RS:0;26d3106abcfe:37277 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:14,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,37277,1732038433903] 2024-11-19T17:47:14,584 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,41933,1732038433950] 2024-11-19T17:47:14,584 DEBUG [RS:1;26d3106abcfe:41933 {}] zookeeper.ZKUtil(111): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,584 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,585 WARN [RS:1;26d3106abcfe:41933 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:14,585 INFO [RS:1;26d3106abcfe:41933 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:14,585 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,595 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:47:14,599 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:47:14,600 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:47:14,607 INFO [RS:1;26d3106abcfe:41933 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:47:14,607 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,608 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:47:14,608 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:47:14,609 INFO [RS:0;26d3106abcfe:37277 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:47:14,609 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,609 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:47:14,609 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:47:14,609 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,609 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,609 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,610 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:14,610 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:1;26d3106abcfe:41933 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:14,611 DEBUG [RS:0;26d3106abcfe:37277 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:14,613 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,613 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,614 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,614 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41933,1732038433950-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:14,614 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,37277,1732038433903-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:14,633 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:47:14,633 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,37277,1732038433903-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,633 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,633 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.Replication(171): 26d3106abcfe,37277,1732038433903 started 2024-11-19T17:47:14,633 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:47:14,634 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,41933,1732038433950-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,634 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,634 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.Replication(171): 26d3106abcfe,41933,1732038433950 started 2024-11-19T17:47:14,648 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,648 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,37277,1732038433903, RpcServer on 26d3106abcfe/172.17.0.2:37277, sessionid=0x101543517660001 2024-11-19T17:47:14,648 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:47:14,648 DEBUG [RS:0;26d3106abcfe:37277 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,648 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,37277,1732038433903' 2024-11-19T17:47:14,648 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:47:14,649 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,37277,1732038433903 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,37277,1732038433903' 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:47:14,650 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:47:14,651 DEBUG [RS:0;26d3106abcfe:37277 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:47:14,651 INFO [RS:0;26d3106abcfe:37277 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:47:14,651 INFO [RS:0;26d3106abcfe:37277 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:47:14,654 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:14,654 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,41933,1732038433950, RpcServer on 26d3106abcfe/172.17.0.2:41933, sessionid=0x101543517660002 2024-11-19T17:47:14,655 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:47:14,655 DEBUG [RS:1;26d3106abcfe:41933 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,655 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,41933,1732038433950' 2024-11-19T17:47:14,655 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:47:14,655 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,41933,1732038433950' 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:47:14,656 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:47:14,657 DEBUG [RS:1;26d3106abcfe:41933 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:47:14,657 INFO [RS:1;26d3106abcfe:41933 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:47:14,657 INFO [RS:1;26d3106abcfe:41933 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:47:14,680 WARN [26d3106abcfe:44853 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-19T17:47:14,752 INFO [RS:0;26d3106abcfe:37277 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:14,753 INFO [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C37277%2C1732038433903, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,37277,1732038433903, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:14,758 INFO [RS:1;26d3106abcfe:41933 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:14,759 INFO [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C41933%2C1732038433950, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:14,778 INFO [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,37277,1732038433903/26d3106abcfe%2C37277%2C1732038433903.1732038434755 2024-11-19T17:47:14,778 DEBUG [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:14,782 INFO [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950/26d3106abcfe%2C41933%2C1732038433950.1732038434760 2024-11-19T17:47:14,782 DEBUG [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:14,931 DEBUG [26d3106abcfe:44853 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-19T17:47:14,931 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(204): Hosts are {26d3106abcfe=0} racks are {/default-rack=0} 2024-11-19T17:47:14,932 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T17:47:14,932 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T17:47:14,932 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T17:47:14,933 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T17:47:14,933 INFO [26d3106abcfe:44853 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T17:47:14,933 INFO [26d3106abcfe:44853 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T17:47:14,933 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T17:47:14,933 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:14,935 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,41933,1732038433950, state=OPENING 2024-11-19T17:47:14,957 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-19T17:47:14,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:14,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:14,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:14,970 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:14,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:14,971 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,41933,1732038433950}] 2024-11-19T17:47:15,124 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:47:15,127 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43143, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:47:15,129 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T17:47:15,130 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:15,130 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T17:47:15,131 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C41933%2C1732038433950.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:15,157 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950/26d3106abcfe%2C41933%2C1732038433950.meta.1732038435132.meta 2024-11-19T17:47:15,158 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:15,159 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:15,159 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T17:47:15,159 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T17:47:15,160 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T17:47:15,160 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T17:47:15,160 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:15,160 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T17:47:15,160 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T17:47:15,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T17:47:15,163 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:47:15,163 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:15,163 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T17:47:15,164 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:47:15,164 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:15,164 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T17:47:15,165 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:47:15,165 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,165 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:15,166 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T17:47:15,166 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:47:15,167 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,167 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:15,167 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:47:15,168 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:15,168 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:15,169 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:47:15,169 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:47:15,169 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:15,170 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:47:15,171 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67095682, jitterRate=-1.964271068572998E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:15,171 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T17:47:15,172 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732038435160Writing region info on filesystem at 1732038435161 (+1 ms)Initializing all the Stores at 1732038435161Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038435161Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038435161Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038435161Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038435161Cleaning up temporary data from old regions at 1732038435169 (+8 ms)Running coprocessor post-open hooks at 1732038435171 (+2 ms)Region opened successfully at 1732038435172 (+1 ms) 2024-11-19T17:47:15,173 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732038435124 2024-11-19T17:47:15,176 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:15,177 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,41933,1732038433950, state=OPEN 2024-11-19T17:47:15,178 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T17:47:15,178 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T17:47:15,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:15,188 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:15,188 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:15,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:15,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:15,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:15,189 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:15,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-19T17:47:15,193 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,41933,1732038433950 in 219 msec 2024-11-19T17:47:15,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-19T17:47:15,196 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 665 msec 2024-11-19T17:47:15,197 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-19T17:47:15,197 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-19T17:47:15,200 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:47:15,200 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1] 2024-11-19T17:47:15,200 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:47:15,202 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34451, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:47:15,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 784 msec 2024-11-19T17:47:15,210 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732038435210, completionTime=-1 2024-11-19T17:47:15,210 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-19T17:47:15,210 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-19T17:47:15,212 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-19T17:47:15,212 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732038495212 2024-11-19T17:47:15,213 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732038555213 2024-11-19T17:47:15,213 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-19T17:47:15,213 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-19T17:47:15,213 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-26d3106abcfe:44853, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,214 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,217 DEBUG [master/26d3106abcfe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-19T17:47:15,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.178sec 2024-11-19T17:47:15,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-19T17:47:15,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-19T17:47:15,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-19T17:47:15,219 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-19T17:47:15,220 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-19T17:47:15,220 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:15,220 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-19T17:47:15,220 DEBUG [master/26d3106abcfe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-19T17:47:15,221 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-19T17:47:15,221 INFO [master/26d3106abcfe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,44853,1732038433738-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:15,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3076c0d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:15,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 26d3106abcfe,44853,-1 for getting cluster id 2024-11-19T17:47:15,288 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-19T17:47:15,290 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b8de8b75-c499-49a7-86c7-aaeb7f384aa7' 2024-11-19T17:47:15,291 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-19T17:47:15,291 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b8de8b75-c499-49a7-86c7-aaeb7f384aa7" 2024-11-19T17:47:15,291 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f0bec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:15,291 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [26d3106abcfe,44853,-1] 2024-11-19T17:47:15,292 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-19T17:47:15,304 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:15,308 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-19T17:47:15,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-19T17:47:15,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:47:15,313 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1] 2024-11-19T17:47:15,314 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:47:15,319 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47938, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:47:15,322 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=26d3106abcfe,44853,1732038433738 2024-11-19T17:47:15,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-19T17:47:15,325 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 26d3106abcfe,44853,1732038433738 2024-11-19T17:47:15,325 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@27fced3a 2024-11-19T17:47:15,325 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-19T17:47:15,327 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-19T17:47:15,329 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732038435323', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-19T17:47:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 2024-11-19T17:47:15,331 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-19T17:47:15,331 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732038435323" procId is: 4 2024-11-19T17:47:15,333 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-19T17:47:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:15,350 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f875107eb3f893450fd5169f943293a, NAME => 'testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732038435323', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1722): Closing 1f875107eb3f893450fd5169f943293a, disabling compactions & flushes 2024-11-19T17:47:15,358 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. after waiting 0 ms 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,358 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,358 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732038435323-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f875107eb3f893450fd5169f943293a: Waiting for close lock at 1732038435358Disabling compacts and flushes for region at 1732038435358Disabling writes for close at 1732038435358Writing region close event to WAL at 1732038435358Closed at 1732038435358 2024-11-19T17:47:15,360 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_ADD_TO_META 2024-11-19T17:47:15,360 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732038435360"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732038435360"}]},"ts":"1732038435360"} 2024-11-19T17:47:15,364 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-19T17:47:15,365 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-19T17:47:15,365 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732038435323","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038435365"}]},"ts":"1732038435365"} 2024-11-19T17:47:15,368 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732038435323, state=ENABLING in hbase:meta 2024-11-19T17:47:15,368 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {26d3106abcfe=0} racks are {/default-rack=0} 2024-11-19T17:47:15,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T17:47:15,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T17:47:15,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T17:47:15,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T17:47:15,369 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T17:47:15,369 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T17:47:15,369 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T17:47:15,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN}] 2024-11-19T17:47:15,371 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN 2024-11-19T17:47:15,371 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN; state=OFFLINE, location=26d3106abcfe,41933,1732038433950; forceNewPlan=false, retain=false 2024-11-19T17:47:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:15,522 INFO [26d3106abcfe:44853 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-19T17:47:15,522 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f875107eb3f893450fd5169f943293a, regionState=OPENING, regionLocation=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:15,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN because future has completed 2024-11-19T17:47:15,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,41933,1732038433950}] 2024-11-19T17:47:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:15,679 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,680 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f875107eb3f893450fd5169f943293a, NAME => 'testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:15,680 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732038435323 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,680 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:15,680 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,680 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,681 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,684 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f875107eb3f893450fd5169f943293a columnFamilyName family 2024-11-19T17:47:15,684 DEBUG [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:15,684 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] regionserver.HStore(327): Store=1f875107eb3f893450fd5169f943293a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:15,684 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,685 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,685 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,685 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,685 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,686 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,691 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-19T17:47:15,691 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1f875107eb3f893450fd5169f943293a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67520550, jitterRate=0.006134599447250366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T17:47:15,692 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:15,693 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1f875107eb3f893450fd5169f943293a: Running coprocessor pre-open hook at 1732038435680Writing region info on filesystem at 1732038435680Initializing all the Stores at 1732038435680Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038435681 (+1 ms)Cleaning up temporary data from old regions at 1732038435685 (+4 ms)Running coprocessor post-open hooks at 1732038435692 (+7 ms)Region opened successfully at 1732038435692 2024-11-19T17:47:15,693 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a., pid=6, masterSystemTime=1732038435677 2024-11-19T17:47:15,696 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,696 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:15,697 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1f875107eb3f893450fd5169f943293a, regionState=OPEN, openSeqNum=2, regionLocation=26d3106abcfe,41933,1732038433950 2024-11-19T17:47:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,41933,1732038433950 because future has completed 2024-11-19T17:47:15,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-19T17:47:15,704 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,41933,1732038433950 in 176 msec 2024-11-19T17:47:15,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-19T17:47:15,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN in 335 msec 2024-11-19T17:47:15,707 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-19T17:47:15,708 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732038435323","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732038435707"}]},"ts":"1732038435707"} 2024-11-19T17:47:15,710 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732038435323, state=ENABLED in hbase:meta 2024-11-19T17:47:15,711 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 execute state=CREATE_TABLE_POST_OPERATION 2024-11-19T17:47:15,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732038435323 in 382 msec 2024-11-19T17:47:15,813 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-19T17:47:15,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44853 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-19T17:47:15,967 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732038435323 completed 2024-11-19T17:47:15,967 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732038435323 get assigned. Timeout = 60000ms 2024-11-19T17:47:15,967 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T17:47:15,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732038435323 assigned to meta. Checking AM states. 2024-11-19T17:47:15,976 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-19T17:47:15,977 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732038435323 assigned. 2024-11-19T17:47:15,981 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732038435323', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a., hostname=26d3106abcfe,41933,1732038433950, seqNum=2] 2024-11-19T17:47:15,988 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f875107eb3f893450fd5169f943293a 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-19T17:47:15,997 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:16,012 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/.tmp/family/f65b7223e5194ce1846260eebbf92d1d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-19T17:47:16,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/.tmp/family/f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:16,016 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/.tmp/family/f65b7223e5194ce1846260eebbf92d1d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:16,017 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, delay=943 ms 2024-11-19T17:47:16,019 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-19T17:47:16,019 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1f875107eb3f893450fd5169f943293a in 31ms, sequenceid=6, compaction requested=false 2024-11-19T17:47:16,019 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732038435323' 2024-11-19T17:47:16,020 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f875107eb3f893450fd5169f943293a: 2024-11-19T17:47:16,760 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-19T17:47:16,761 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-19T17:47:16,765 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732038435323 2024-11-19T17:47:16,765 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732038435323 Metrics about Tables on a single HBase RegionServer 2024-11-19T17:47:16,767 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:47:16,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-19T17:47:16,768 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-19T17:47:16,768 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-19T17:47:16,962 DEBUG [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f65b7223e5194ce1846260eebbf92d1d_0. 2024-11-19T17:47:16,963 DEBUG [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f65b7223e5194ce1846260eebbf92d1d_117. 2024-11-19T17:47:16,964 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, with total blocks 2, and data blocks 1 2024-11-19T17:47:16,964 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-19T17:47:16,964 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-19T17:47:16,964 DEBUG [hfile-prefetch-1732038431515 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d. Will sleep 100ms and try the verification again. 2024-11-19T17:47:17,021 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;26d3106abcfe:41933,5,FailOnTimeoutGroup] 2024-11-19T17:47:17,021 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,41933,1732038433950' ***** 2024-11-19T17:47:17,021 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-19T17:47:17,022 INFO [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T17:47:17,042 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T17:47:17,042 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T17:47:17,042 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T17:47:17,043 INFO [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence 2024-11-19T17:47:17,043 INFO [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T17:47:17,043 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T17:47:17,053 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1487): Thread RS:1;26d3106abcfe:41933 started persisting bucket cache to file 2024-11-19T17:47:17,053 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:17,053 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:17,057 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-19T17:47:17,057 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732038437053 2024-11-19T17:47:17,057 DEBUG [RS:1;26d3106abcfe:41933 {}] bucket.BucketCache(1509): Thread RS:1;26d3106abcfe:41933 finished persisting bucket cache to file, renaming 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:47:17,058 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,41933,1732038433950 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:17,058 INFO [RS:1;26d3106abcfe:41933 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;26d3106abcfe:41933. 2024-11-19T17:47:17,058 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:17,058 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f875107eb3f893450fd5169f943293a, disabling compactions & flushes 2024-11-19T17:47:17,058 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:17,059 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:17,059 DEBUG [RS:1;26d3106abcfe:41933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:17,059 DEBUG [RS:1;26d3106abcfe:41933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:17,059 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. after waiting 0 ms 2024-11-19T17:47:17,059 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:17,059 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:47:17,059 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:47:17,059 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:47:17,059 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T17:47:17,059 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T17:47:17,059 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1f875107eb3f893450fd5169f943293a=testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.} 2024-11-19T17:47:17,059 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:17,059 DEBUG [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:17,059 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:47:17,060 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:47:17,060 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:47:17,060 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:47:17,060 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:47:17,060 DEBUG [StoreFileCloser-1f875107eb3f893450fd5169f943293a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:17,060 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-19T17:47:17,060 WARN [hfile-prefetch-1732038431515 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-19T17:47:17,060 DEBUG [hfile-prefetch-1732038431515 {}] hfile.PrefetchExecutor(121): Prefetch completed for f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:17,065 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-19T17:47:17,065 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:17,065 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f875107eb3f893450fd5169f943293a: Waiting for close lock at 1732038437058Running coprocessor pre-close hooks at 1732038437058Disabling compacts and flushes for region at 1732038437058Disabling writes for close at 1732038437059 (+1 ms)Writing region close event to WAL at 1732038437060 (+1 ms)Running coprocessor post-close hooks at 1732038437065 (+5 ms)Closed at 1732038437065 2024-11-19T17:47:17,066 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:17,071 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:17,084 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/99244bcb7bf24c86907a0aa4ba1a2259 is 209, key is testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a./info:regioninfo/1732038435697/Put/seqid=0 2024-11-19T17:47:17,085 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:17,094 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:17,107 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/88300d0bb08540d1843fd4235ae90dbe is 43, key is default/ns:d/1732038435203/Put/seqid=0 2024-11-19T17:47:17,108 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:17,119 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:17,121 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:17,125 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-19T17:47:17,125 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-19T17:47:17,125 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:17,126 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:17,130 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:17,130 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038437125 2024-11-19T17:47:17,130 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-19T17:47:17,137 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/3ca7c8275c9a4b3da930dddfa881fea3 is 80, key is testBlockEvictionOnGracefulStop1732038435323/table:state/1732038435707/Put/seqid=0 2024-11-19T17:47:17,138 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:17,140 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/99244bcb7bf24c86907a0aa4ba1a2259 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:17,142 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259, delay=953 ms 2024-11-19T17:47:17,143 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259, entries=10, sequenceid=11, filesize=7.0 K 2024-11-19T17:47:17,143 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/ns/88300d0bb08540d1843fd4235ae90dbe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:17,145 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe, delay=973 ms 2024-11-19T17:47:17,147 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe, entries=2, sequenceid=11, filesize=5.0 K 2024-11-19T17:47:17,147 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/table/3ca7c8275c9a4b3da930dddfa881fea3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:17,149 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3, delay=926 ms 2024-11-19T17:47:17,150 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3, entries=2, sequenceid=11, filesize=5.3 K 2024-11-19T17:47:17,151 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false 2024-11-19T17:47:17,152 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:17,152 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:17,153 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:17,158 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-19T17:47:17,158 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:47:17,158 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:17,158 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038437059Running coprocessor pre-close hooks at 1732038437059Disabling compacts and flushes for region at 1732038437059Disabling writes for close at 1732038437060 (+1 ms)Obtaining lock to block concurrent updates at 1732038437060Preparing flush snapshotting stores in 1588230740 at 1732038437060Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732038437061 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732038437061Flushing 1588230740/info: creating writer at 1732038437061Flushing 1588230740/info: appending metadata at 1732038437083 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732038437083Flushing 1588230740/ns: creating writer at 1732038437088 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732038437107 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732038437107Flushing 1588230740/table: creating writer at 1732038437110 (+3 ms)Flushing 1588230740/table: appending metadata at 1732038437137 (+27 ms)Flushing 1588230740/table: closing flushed file at 1732038437137Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68c74de0: reopening flushed file at 1732038437140 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5345aa6d: reopening flushed file at 1732038437143 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46357b33: reopening flushed file at 1732038437147 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=11, compaction requested=false at 1732038437151 (+4 ms)Writing region close event to WAL at 1732038437153 (+2 ms)Running coprocessor post-close hooks at 1732038437158 (+5 ms)Closed at 1732038437158 2024-11-19T17:47:17,159 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:17,260 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,41933,1732038433950; all regions closed. 2024-11-19T17:47:17,261 DEBUG [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:17,262 INFO [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C41933%2C1732038433950.meta:.meta(num 1732038435132) 2024-11-19T17:47:17,263 DEBUG [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:17,263 INFO [RS:1;26d3106abcfe:41933 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C41933%2C1732038433950:(num 1732038434760) 2024-11-19T17:47:17,263 DEBUG [RS:1;26d3106abcfe:41933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:17,263 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:17,263 INFO [RS:1;26d3106abcfe:41933 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:17,263 INFO [RS:1;26d3106abcfe:41933 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:17,263 INFO [RS:1;26d3106abcfe:41933 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:17,263 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:17,264 INFO [RS:1;26d3106abcfe:41933 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41933 2024-11-19T17:47:17,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,41933,1732038433950 2024-11-19T17:47:17,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:17,283 INFO [RS:1;26d3106abcfe:41933 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:17,294 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,41933,1732038433950] 2024-11-19T17:47:17,304 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,41933,1732038433950 already deleted, retry=false 2024-11-19T17:47:17,305 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 26d3106abcfe,41933,1732038433950 on 26d3106abcfe,44853,1732038433738 2024-11-19T17:47:17,314 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 26d3106abcfe,41933,1732038433950, splitWal=true, meta=true 2024-11-19T17:47:17,318 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 26d3106abcfe,41933,1732038433950, splitWal=true, meta=true 2024-11-19T17:47:17,318 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 26d3106abcfe,41933,1732038433950 (carryingMeta=true) 26d3106abcfe,41933,1732038433950/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@359aa607[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-19T17:47:17,319 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 26d3106abcfe,41933,1732038433950, splitWal=true, meta=true, isMeta: true 2024-11-19T17:47:17,319 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950-splitting 2024-11-19T17:47:17,320 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950-splitting dir is empty, no logs to split. 2024-11-19T17:47:17,320 INFO [PEWorker-2 {}] master.SplitWALManager(105): 26d3106abcfe,41933,1732038433950 WAL count=0, meta=true 2024-11-19T17:47:17,322 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950-splitting dir is empty, no logs to split. 2024-11-19T17:47:17,322 INFO [PEWorker-2 {}] master.SplitWALManager(105): 26d3106abcfe,41933,1732038433950 WAL count=0, meta=true 2024-11-19T17:47:17,322 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 26d3106abcfe,41933,1732038433950 WAL splitting is done? wals=0, meta=true 2024-11-19T17:47:17,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-19T17:47:17,325 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:17,325 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T17:47:17,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:17,394 INFO [RS:1;26d3106abcfe:41933 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:17,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41933-0x101543517660002, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:17,394 INFO [RS:1;26d3106abcfe:41933 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,41933,1732038433950; zookeeper connection closed. 2024-11-19T17:47:17,394 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4b23a3fb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4b23a3fb 2024-11-19T17:47:17,476 DEBUG [26d3106abcfe:44853 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-19T17:47:17,476 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:17,477 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,37277,1732038433903, state=OPENING 2024-11-19T17:47:17,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:17,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:17,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:17,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:17,488 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-19T17:47:17,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,37277,1732038433903}] 2024-11-19T17:47:17,540 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/26d3106abcfe:0 server-side Connection retries=45 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-19T17:47:17,540 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-19T17:47:17,541 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-19T17:47:17,541 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42569 2024-11-19T17:47:17,543 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42569 connecting to ZooKeeper ensemble=127.0.0.1:64906 2024-11-19T17:47:17,544 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:17,544 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:17,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:425690x0, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-19T17:47:17,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:425690x0, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-19T17:47:17,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42569-0x101543517660003 connected 2024-11-19T17:47:17,576 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-19T17:47:17,576 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-19T17:47:17,577 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache 2024-11-19T17:47:17,577 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-19T17:47:17,577 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-19T17:47:17,577 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-19T17:47:17,577 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-19T17:47:17,577 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-19T17:47:17,578 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-19T17:47:17,580 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-19T17:47:17,587 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-19T17:47:17,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-19T17:47:17,591 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-19T17:47:17,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-19T17:47:17,591 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-19T17:47:17,591 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42569 2024-11-19T17:47:17,592 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-19T17:47:17,592 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence is complete. 2024-11-19T17:47:17,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42569 2024-11-19T17:47:17,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42569 2024-11-19T17:47:17,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42569 2024-11-19T17:47:17,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42569 2024-11-19T17:47:17,621 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(746): ClusterId : b8de8b75-c499-49a7-86c7-aaeb7f384aa7 2024-11-19T17:47:17,622 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-19T17:47:17,632 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-19T17:47:17,632 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-19T17:47:17,642 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-19T17:47:17,643 DEBUG [RS:2;26d3106abcfe:42569 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@635fba9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=26d3106abcfe/172.17.0.2:0 2024-11-19T17:47:17,649 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-19T17:47:17,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-19T17:47:17,656 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-19T17:47:17,656 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:17,657 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-19T17:47:17,658 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C37277%2C1732038433903.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,37277,1732038433903, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:17,664 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;26d3106abcfe:42569 2024-11-19T17:47:17,664 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-19T17:47:17,664 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-19T17:47:17,664 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-19T17:47:17,665 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(2659): reportForDuty to master=26d3106abcfe,44853,1732038433738 with port=42569, startcode=1732038437540 2024-11-19T17:47:17,665 DEBUG [RS:2;26d3106abcfe:42569 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-19T17:47:17,670 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-19T17:47:17,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44853 {}] master.ServerManager(517): Registering regionserver=26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,672 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5 2024-11-19T17:47:17,672 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-19T17:47:17,672 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-19T17:47:17,678 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,37277,1732038433903/26d3106abcfe%2C37277%2C1732038433903.meta.1732038437659.meta 2024-11-19T17:47:17,680 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:17,680 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-19T17:47:17,681 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-19T17:47:17,681 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-19T17:47:17,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-19T17:47:17,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:17,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-19T17:47:17,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:17,684 DEBUG [RS:2;26d3106abcfe:42569 {}] zookeeper.ZKUtil(111): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,684 WARN [RS:2;26d3106abcfe:42569 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-19T17:47:17,684 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [26d3106abcfe,42569,1732038437540] 2024-11-19T17:47:17,684 INFO [RS:2;26d3106abcfe:42569 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-19T17:47:17,684 DEBUG [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,685 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259, delay=970 ms 2024-11-19T17:47:17,686 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:17,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:17,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-19T17:47:17,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-19T17:47:17,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:17,690 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe, delay=978 ms 2024-11-19T17:47:17,691 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:17,691 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:17,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-19T17:47:17,692 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-19T17:47:17,693 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-19T17:47:17,693 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:17,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:17,693 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-19T17:47:17,694 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-19T17:47:17,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:17,695 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-19T17:47:17,696 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3, delay=980 ms 2024-11-19T17:47:17,696 INFO [RS:2;26d3106abcfe:42569 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-19T17:47:17,696 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,697 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:17,697 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-19T17:47:17,697 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-19T17:47:17,698 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:17,698 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740 2024-11-19T17:47:17,698 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-19T17:47:17,699 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-19T17:47:17,699 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-19T17:47:17,700 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-19T17:47:17,700 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65495123, jitterRate=-0.02404661476612091}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-19T17:47:17,700 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-19T17:47:17,701 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-19T17:47:17,701 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732038437682Writing region info on filesystem at 1732038437682Initializing all the Stores at 1732038437682Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038437682Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038437682Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038437682Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732038437682Cleaning up temporary data from old regions at 1732038437699 (+17 ms)Running coprocessor post-open hooks at 1732038437700 (+1 ms)Region opened successfully at 1732038437701 (+1 ms) 2024-11-19T17:47:17,702 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-19T17:47:17,702 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,702 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732038437648 2024-11-19T17:47:17,702 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,702 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,702 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/26d3106abcfe:0, corePoolSize=2, maxPoolSize=2 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/26d3106abcfe:0, corePoolSize=1, maxPoolSize=1 2024-11-19T17:47:17,703 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:17,704 DEBUG [RS:2;26d3106abcfe:42569 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/26d3106abcfe:0, corePoolSize=3, maxPoolSize=3 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,704 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,42569,1732038437540-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-19T17:47:17,705 DEBUG [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-19T17:47:17,705 INFO [RS_OPEN_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-19T17:47:17,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:17,706 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 26d3106abcfe,37277,1732038433903, state=OPEN 2024-11-19T17:47:17,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:17,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:17,715 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-19T17:47:17,715 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:17,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:17,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:17,715 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-19T17:47:17,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-19T17:47:17,717 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=26d3106abcfe,37277,1732038433903 in 227 msec 2024-11-19T17:47:17,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-19T17:47:17,720 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 394 msec 2024-11-19T17:47:17,720 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 26d3106abcfe,41933,1732038433950 had 2 regions 2024-11-19T17:47:17,721 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 26d3106abcfe,41933,1732038433950, splitWal=true, meta=true, isMeta: false 2024-11-19T17:47:17,722 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950-splitting dir is empty, no logs to split. 2024-11-19T17:47:17,722 INFO [PEWorker-3 {}] master.SplitWALManager(105): 26d3106abcfe,41933,1732038433950 WAL count=0, meta=false 2024-11-19T17:47:17,723 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,41933,1732038433950-splitting dir is empty, no logs to split. 2024-11-19T17:47:17,723 INFO [PEWorker-3 {}] master.SplitWALManager(105): 26d3106abcfe,41933,1732038433950 WAL count=0, meta=false 2024-11-19T17:47:17,723 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 26d3106abcfe,41933,1732038433950 WAL splitting is done? wals=0, meta=false 2024-11-19T17:47:17,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN}] 2024-11-19T17:47:17,726 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-19T17:47:17,726 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=26d3106abcfe,42569,1732038437540-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,726 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN 2024-11-19T17:47:17,726 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,727 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.Replication(171): 26d3106abcfe,42569,1732038437540 started 2024-11-19T17:47:17,727 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-19T17:47:17,742 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-19T17:47:17,742 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1482): Serving as 26d3106abcfe,42569,1732038437540, RpcServer on 26d3106abcfe/172.17.0.2:42569, sessionid=0x101543517660003 2024-11-19T17:47:17,742 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-19T17:47:17,742 DEBUG [RS:2;26d3106abcfe:42569 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,742 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,42569,1732038437540' 2024-11-19T17:47:17,743 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-19T17:47:17,743 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 26d3106abcfe,42569,1732038437540 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '26d3106abcfe,42569,1732038437540' 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-19T17:47:17,744 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-19T17:47:17,745 DEBUG [RS:2;26d3106abcfe:42569 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-19T17:47:17,745 INFO [RS:2;26d3106abcfe:42569 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-19T17:47:17,745 INFO [RS:2;26d3106abcfe:42569 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-19T17:47:17,846 INFO [RS:2;26d3106abcfe:42569 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-19T17:47:17,847 INFO [RS:2;26d3106abcfe:42569 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=26d3106abcfe%2C42569%2C1732038437540, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,42569,1732038437540, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs, maxLogs=55 2024-11-19T17:47:17,868 INFO [RS:2;26d3106abcfe:42569 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/WALs/26d3106abcfe,42569,1732038437540/26d3106abcfe%2C42569%2C1732038437540.1732038437848 2024-11-19T17:47:17,868 DEBUG [RS:2;26d3106abcfe:42569 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(204): Hosts are {26d3106abcfe=0} racks are {/default-rack=0} 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-19T17:47:17,878 INFO [26d3106abcfe:44853 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-19T17:47:17,878 INFO [26d3106abcfe:44853 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-19T17:47:17,878 DEBUG [26d3106abcfe:44853 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-19T17:47:17,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1f875107eb3f893450fd5169f943293a, regionState=OPENING, regionLocation=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:17,880 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 26d3106abcfe:41933 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 26d3106abcfe/172.17.0.2:41933 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-19T17:47:17,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1, error=java.net.ConnectException: Call to address=26d3106abcfe:41933 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 26d3106abcfe/172.17.0.2:41933 2024-11-19T17:47:17,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-19T17:47:17,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,41933,1732038433950, seqNum=-1 from cache 2024-11-19T17:47:17,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 26d3106abcfe:41933 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 26d3106abcfe/172.17.0.2:41933 2024-11-19T17:47:17,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-19T17:47:17,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=26d3106abcfe,37277,1732038433903, seqNum=-1] 2024-11-19T17:47:17,985 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-19T17:47:17,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45705, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-19T17:47:17,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN because future has completed 2024-11-19T17:47:17,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,37277,1732038433903}] 2024-11-19T17:47:18,147 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,147 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f875107eb3f893450fd5169f943293a, NAME => 'testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.', STARTKEY => '', ENDKEY => ''} 2024-11-19T17:47:18,149 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732038435323 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,149 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-19T17:47:18,150 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,150 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,152 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,153 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f875107eb3f893450fd5169f943293a columnFamilyName family 2024-11-19T17:47:18,153 DEBUG [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-19T17:47:18,155 DEBUG [StoreFileOpener-1f875107eb3f893450fd5169f943293a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d, delay=923 ms 2024-11-19T17:47:18,157 DEBUG [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:18,158 INFO [StoreOpener-1f875107eb3f893450fd5169f943293a-1 {}] regionserver.HStore(327): Store=1f875107eb3f893450fd5169f943293a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-19T17:47:18,158 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,159 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,160 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,160 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,160 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,161 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,162 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 1f875107eb3f893450fd5169f943293a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64391686, jitterRate=-0.04048910737037659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-19T17:47:18,162 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,163 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 1f875107eb3f893450fd5169f943293a: Running coprocessor pre-open hook at 1732038438150Writing region info on filesystem at 1732038438150Initializing all the Stores at 1732038438151 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732038438151Cleaning up temporary data from old regions at 1732038438160 (+9 ms)Running coprocessor post-open hooks at 1732038438162 (+2 ms)Region opened successfully at 1732038438163 (+1 ms) 2024-11-19T17:47:18,164 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a., pid=11, masterSystemTime=1732038438142 2024-11-19T17:47:18,167 DEBUG [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,168 INFO [RS_OPEN_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1f875107eb3f893450fd5169f943293a, regionState=OPEN, openSeqNum=10, regionLocation=26d3106abcfe,37277,1732038433903 2024-11-19T17:47:18,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,37277,1732038433903 because future has completed 2024-11-19T17:47:18,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-19T17:47:18,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f875107eb3f893450fd5169f943293a, server=26d3106abcfe,37277,1732038433903 in 184 msec 2024-11-19T17:47:18,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-19T17:47:18,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732038435323, region=1f875107eb3f893450fd5169f943293a, ASSIGN in 451 msec 2024-11-19T17:47:18,179 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 26d3106abcfe,41933,1732038433950 after splitting done 2024-11-19T17:47:18,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 26d3106abcfe,41933,1732038433950, splitWal=true, meta=true in 873 msec 2024-11-19T17:47:18,243 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-19T17:47:18,243 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:47:18,243 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:18,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-19T17:47:18,244 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-19T17:47:18,244 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=697722502, stopped=false 2024-11-19T17:47:18,244 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=26d3106abcfe,44853,1732038433738 2024-11-19T17:47:18,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:18,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:18,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-19T17:47:18,299 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:18,299 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:47:18,299 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-19T17:47:18,299 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:18,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,300 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:18,300 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,37277,1732038433903' ***** 2024-11-19T17:47:18,300 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:47:18,300 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '26d3106abcfe,42569,1732038437540' ***** 2024-11-19T17:47:18,300 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-19T17:47:18,300 INFO [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T17:47:18,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:18,300 INFO [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1738): Disabling cache 2024-11-19T17:47:18,300 INFO [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence 2024-11-19T17:47:18,300 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T17:47:18,300 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T17:47:18,300 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-19T17:47:18,301 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-19T17:47:18,301 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T17:47:18,301 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:18,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:18,301 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-19T17:47:18,301 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-19T17:47:18,301 INFO [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T17:47:18,301 INFO [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/dffc6f45-cec5-96f9-736a-338fcf8254f9/bucket.persistence 2024-11-19T17:47:18,301 INFO [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-19T17:47:18,301 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T17:47:18,301 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-19T17:47:18,305 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:47:18,307 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:18,311 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1487): Thread RS:0;26d3106abcfe:37277 started persisting bucket cache to file 2024-11-19T17:47:18,311 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1487): Thread RS:2;26d3106abcfe:42569 started persisting bucket cache to file 2024-11-19T17:47:18,311 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:18,311 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:18,311 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-19T17:47:18,311 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-19T17:47:18,314 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:18,314 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-19T17:47:18,314 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038438311 2024-11-19T17:47:18,314 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732038438311 2024-11-19T17:47:18,314 DEBUG [RS:2;26d3106abcfe:42569 {}] bucket.BucketCache(1509): Thread RS:2;26d3106abcfe:42569 finished persisting bucket cache to file, renaming 2024-11-19T17:47:18,314 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:47:18,314 INFO [RS:2;26d3106abcfe:42569 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:47:18,315 INFO [RS:2;26d3106abcfe:42569 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:47:18,315 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,42569,1732038437540 2024-11-19T17:47:18,315 INFO [RS:2;26d3106abcfe:42569 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:18,315 INFO [RS:2;26d3106abcfe:42569 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;26d3106abcfe:42569. 2024-11-19T17:47:18,315 DEBUG [RS:2;26d3106abcfe:42569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:18,315 DEBUG [RS:2;26d3106abcfe:42569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,315 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,42569,1732038437540; all regions closed. 2024-11-19T17:47:18,316 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-19T17:47:18,317 DEBUG [RS:2;26d3106abcfe:42569 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C42569%2C1732038437540:(num 1732038437848) 2024-11-19T17:47:18,317 DEBUG [RS:2;26d3106abcfe:42569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:47:18,317 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:47:18,317 INFO [RS:2;26d3106abcfe:42569 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:18,318 INFO [RS:2;26d3106abcfe:42569 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42569 2024-11-19T17:47:18,320 INFO [regionserver/26d3106abcfe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:18,329 DEBUG [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1509): Thread RS:0;26d3106abcfe:37277 finished persisting bucket cache to file, renaming 2024-11-19T17:47:18,329 WARN [RS:0;26d3106abcfe:37277 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-19T17:47:18,329 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-19T17:47:18,329 INFO [RS:0;26d3106abcfe:37277 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-19T17:47:18,329 INFO [RS:0;26d3106abcfe:37277 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-19T17:47:18,329 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(959): stopping server 26d3106abcfe,37277,1732038433903 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;26d3106abcfe:37277. 2024-11-19T17:47:18,330 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:18,330 DEBUG [RS:0;26d3106abcfe:37277 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-19T17:47:18,330 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f875107eb3f893450fd5169f943293a, disabling compactions & flushes 2024-11-19T17:47:18,330 DEBUG [RS:0;26d3106abcfe:37277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,330 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,330 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-19T17:47:18,330 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. after waiting 0 ms 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-19T17:47:18,330 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-19T17:47:18,330 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-19T17:47:18,331 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-19T17:47:18,331 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1f875107eb3f893450fd5169f943293a=testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a.} 2024-11-19T17:47:18,331 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-19T17:47:18,331 DEBUG [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1f875107eb3f893450fd5169f943293a 2024-11-19T17:47:18,331 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-19T17:47:18,331 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-19T17:47:18,331 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-19T17:47:18,331 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-19T17:47:18,331 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-19T17:47:18,331 DEBUG [StoreFileCloser-1f875107eb3f893450fd5169f943293a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/family/f65b7223e5194ce1846260eebbf92d1d 2024-11-19T17:47:18,331 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-19T17:47:18,336 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/default/testBlockEvictionOnGracefulStop1732038435323/1f875107eb3f893450fd5169f943293a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-19T17:47:18,337 INFO [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,337 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f875107eb3f893450fd5169f943293a: Waiting for close lock at 1732038438330Running coprocessor pre-close hooks at 1732038438330Disabling compacts and flushes for region at 1732038438330Disabling writes for close at 1732038438330Writing region close event to WAL at 1732038438332 (+2 ms)Running coprocessor post-close hooks at 1732038438337 (+5 ms)Closed at 1732038438337 2024-11-19T17:47:18,337 DEBUG [RS_CLOSE_REGION-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a. 2024-11-19T17:47:18,343 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c60ba55a5a434ff2a66bc0864b0174f5 2024-11-19T17:47:18,357 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/c60ba55a5a434ff2a66bc0864b0174f5 is 209, key is testBlockEvictionOnGracefulStop1732038435323,,1732038435328.1f875107eb3f893450fd5169f943293a./info:regioninfo/1732038438168/Put/seqid=0 2024-11-19T17:47:18,358 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/c60ba55a5a434ff2a66bc0864b0174f5 2024-11-19T17:47:18,360 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/.tmp/info/c60ba55a5a434ff2a66bc0864b0174f5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/c60ba55a5a434ff2a66bc0864b0174f5 2024-11-19T17:47:18,361 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/c60ba55a5a434ff2a66bc0864b0174f5, delay=919 ms 2024-11-19T17:47:18,362 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/c60ba55a5a434ff2a66bc0864b0174f5, entries=8, sequenceid=19, filesize=6.6 K 2024-11-19T17:47:18,363 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false 2024-11-19T17:47:18,364 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/99244bcb7bf24c86907a0aa4ba1a2259 2024-11-19T17:47:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,42569,1732038437540 2024-11-19T17:47:18,364 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:18,364 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/info/c60ba55a5a434ff2a66bc0864b0174f5 2024-11-19T17:47:18,364 INFO [RS:2;26d3106abcfe:42569 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:18,364 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/ns/88300d0bb08540d1843fd4235ae90dbe 2024-11-19T17:47:18,365 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/table/3ca7c8275c9a4b3da930dddfa881fea3 2024-11-19T17:47:18,369 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-19T17:47:18,369 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-19T17:47:18,369 INFO [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:18,369 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732038438331Running coprocessor pre-close hooks at 1732038438331Disabling compacts and flushes for region at 1732038438331Disabling writes for close at 1732038438331Obtaining lock to block concurrent updates at 1732038438331Preparing flush snapshotting stores in 1588230740 at 1732038438331Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732038438332 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732038438332Flushing 1588230740/info: creating writer at 1732038438332Flushing 1588230740/info: appending metadata at 1732038438357 (+25 ms)Flushing 1588230740/info: closing flushed file at 1732038438357Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6558873b: reopening flushed file at 1732038438360 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 32ms, sequenceid=19, compaction requested=false at 1732038438363 (+3 ms)Writing region close event to WAL at 1732038438365 (+2 ms)Running coprocessor post-close hooks at 1732038438369 (+4 ms)Closed at 1732038438369 2024-11-19T17:47:18,369 DEBUG [RS_CLOSE_META-regionserver/26d3106abcfe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-19T17:47:18,383 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,42569,1732038437540] 2024-11-19T17:47:18,393 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,42569,1732038437540 already deleted, retry=false 2024-11-19T17:47:18,393 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26d3106abcfe,42569,1732038437540 expired; onlineServers=1 2024-11-19T17:47:18,483 INFO [RS:2;26d3106abcfe:42569 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:18,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42569-0x101543517660003, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,483 INFO [RS:2;26d3106abcfe:42569 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,42569,1732038437540; zookeeper connection closed. 2024-11-19T17:47:18,483 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@55cbb5fe {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@55cbb5fe 2024-11-19T17:47:18,531 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(976): stopping server 26d3106abcfe,37277,1732038433903; all regions closed. 2024-11-19T17:47:18,533 DEBUG [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:18,533 INFO [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C37277%2C1732038433903.meta:.meta(num 1732038437659) 2024-11-19T17:47:18,535 DEBUG [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/oldWALs 2024-11-19T17:47:18,535 INFO [RS:0;26d3106abcfe:37277 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 26d3106abcfe%2C37277%2C1732038433903:(num 1732038434755) 2024-11-19T17:47:18,535 DEBUG [RS:0;26d3106abcfe:37277 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-19T17:47:18,535 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.LeaseManager(133): Closed leases 2024-11-19T17:47:18,535 INFO [RS:0;26d3106abcfe:37277 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:18,536 INFO [RS:0;26d3106abcfe:37277 {}] hbase.ChoreService(370): Chore service for: regionserver/26d3106abcfe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:18,536 INFO [RS:0;26d3106abcfe:37277 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:18,536 INFO [regionserver/26d3106abcfe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:18,536 INFO [RS:0;26d3106abcfe:37277 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37277 2024-11-19T17:47:18,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/26d3106abcfe,37277,1732038433903 2024-11-19T17:47:18,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-19T17:47:18,551 INFO [RS:0;26d3106abcfe:37277 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:18,562 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [26d3106abcfe,37277,1732038433903] 2024-11-19T17:47:18,572 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/26d3106abcfe,37277,1732038433903 already deleted, retry=false 2024-11-19T17:47:18,572 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 26d3106abcfe,37277,1732038433903 expired; onlineServers=0 2024-11-19T17:47:18,572 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '26d3106abcfe,44853,1732038433738' ***** 2024-11-19T17:47:18,572 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-19T17:47:18,572 INFO [M:0;26d3106abcfe:44853 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-19T17:47:18,572 INFO [M:0;26d3106abcfe:44853 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-19T17:47:18,573 DEBUG [M:0;26d3106abcfe:44853 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-19T17:47:18,573 DEBUG [M:0;26d3106abcfe:44853 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-19T17:47:18,573 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-19T17:47:18,573 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038434451 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.small.0-1732038434451,5,FailOnTimeoutGroup] 2024-11-19T17:47:18,573 DEBUG [master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038434450 {}] cleaner.HFileCleaner(306): Exit Thread[master/26d3106abcfe:0:becomeActiveMaster-HFileCleaner.large.0-1732038434450,5,FailOnTimeoutGroup] 2024-11-19T17:47:18,573 INFO [M:0;26d3106abcfe:44853 {}] hbase.ChoreService(370): Chore service for: master/26d3106abcfe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-19T17:47:18,573 INFO [M:0;26d3106abcfe:44853 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-19T17:47:18,573 DEBUG [M:0;26d3106abcfe:44853 {}] master.HMaster(1795): Stopping service threads 2024-11-19T17:47:18,573 INFO [M:0;26d3106abcfe:44853 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-19T17:47:18,573 INFO [M:0;26d3106abcfe:44853 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-19T17:47:18,573 INFO [M:0;26d3106abcfe:44853 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-19T17:47:18,574 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-19T17:47:18,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-19T17:47:18,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-19T17:47:18,582 DEBUG [M:0;26d3106abcfe:44853 {}] zookeeper.ZKUtil(347): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-19T17:47:18,583 WARN [M:0;26d3106abcfe:44853 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-19T17:47:18,583 INFO [M:0;26d3106abcfe:44853 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/.lastflushedseqids 2024-11-19T17:47:18,588 INFO [M:0;26d3106abcfe:44853 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-19T17:47:18,588 INFO [M:0;26d3106abcfe:44853 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-19T17:47:18,589 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-19T17:47:18,589 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:18,589 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:18,589 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-19T17:47:18,589 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:18,589 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-19T17:47:18,600 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ae543cbd1ea4aa588ed99f463917051 2024-11-19T17:47:18,615 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ae543cbd1ea4aa588ed99f463917051 is 82, key is hbase:meta,,1/info:regioninfo/1732038437705/Put/seqid=0 2024-11-19T17:47:18,616 INFO [M:0;26d3106abcfe:44853 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ae543cbd1ea4aa588ed99f463917051 2024-11-19T17:47:18,622 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 16484b8fd386488db91235202eb8d8ff 2024-11-19T17:47:18,635 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16484b8fd386488db91235202eb8d8ff is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732038435712/Put/seqid=0 2024-11-19T17:47:18,635 INFO [M:0;26d3106abcfe:44853 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16484b8fd386488db91235202eb8d8ff 2024-11-19T17:47:18,642 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2ef37c4782154d7986fe9cb7c10ad105 2024-11-19T17:47:18,660 DEBUG [M:0;26d3106abcfe:44853 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ef37c4782154d7986fe9cb7c10ad105 is 69, key is 26d3106abcfe,37277,1732038433903/rs:state/1732038434540/Put/seqid=0 2024-11-19T17:47:18,661 INFO [M:0;26d3106abcfe:44853 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ef37c4782154d7986fe9cb7c10ad105 2024-11-19T17:47:18,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,662 INFO [RS:0;26d3106abcfe:37277 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:18,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37277-0x101543517660001, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,662 INFO [RS:0;26d3106abcfe:37277 {}] regionserver.HRegionServer(1031): Exiting; stopping=26d3106abcfe,37277,1732038433903; zookeeper connection closed. 2024-11-19T17:47:18,662 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@103b9ba2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@103b9ba2 2024-11-19T17:47:18,662 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-19T17:47:18,662 INFO [M:0;26d3106abcfe:44853 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ef37c4782154d7986fe9cb7c10ad105 2024-11-19T17:47:18,663 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8ae543cbd1ea4aa588ed99f463917051 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ae543cbd1ea4aa588ed99f463917051 2024-11-19T17:47:18,664 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8ae543cbd1ea4aa588ed99f463917051, entries=14, sequenceid=102, filesize=5.9 K 2024-11-19T17:47:18,665 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/16484b8fd386488db91235202eb8d8ff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/16484b8fd386488db91235202eb8d8ff 2024-11-19T17:47:18,666 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/16484b8fd386488db91235202eb8d8ff, entries=11, sequenceid=102, filesize=6.5 K 2024-11-19T17:47:18,666 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2ef37c4782154d7986fe9cb7c10ad105 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ef37c4782154d7986fe9cb7c10ad105 2024-11-19T17:47:18,667 INFO [M:0;26d3106abcfe:44853 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2ef37c4782154d7986fe9cb7c10ad105 2024-11-19T17:47:18,667 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/a9220475-05f9-621e-ca5a-c8ce5cbe07a8/ce2446c3-f8b1-74bd-3197-8d44fca7a4f5/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2ef37c4782154d7986fe9cb7c10ad105, entries=3, sequenceid=102, filesize=5.3 K 2024-11-19T17:47:18,668 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false 2024-11-19T17:47:18,670 INFO [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-19T17:47:18,670 DEBUG [M:0;26d3106abcfe:44853 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732038438589Disabling compacts and flushes for region at 1732038438589Disabling writes for close at 1732038438589Obtaining lock to block concurrent updates at 1732038438589Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732038438589Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732038438589Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732038438590 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732038438590Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732038438615 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732038438615Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732038438617 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732038438634 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732038438634Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732038438637 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732038438659 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732038438659Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f562d44: reopening flushed file at 1732038438663 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a720d0e: reopening flushed file at 1732038438664 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1e2df8: reopening flushed file at 1732038438666 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=102, compaction requested=false at 1732038438668 (+2 ms)Writing region close event to WAL at 1732038438670 (+2 ms)Closed at 1732038438670 2024-11-19T17:47:18,671 INFO [M:0;26d3106abcfe:44853 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-19T17:47:18,671 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-19T17:47:18,671 INFO [M:0;26d3106abcfe:44853 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44853 2024-11-19T17:47:18,671 INFO [M:0;26d3106abcfe:44853 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-19T17:47:18,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,783 INFO [M:0;26d3106abcfe:44853 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-19T17:47:18,783 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44853-0x101543517660000, quorum=127.0.0.1:64906, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-19T17:47:18,789 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-19T17:47:18,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-19T17:47:18,792 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-19T17:47:18,798 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=528 (was 557), ProcessCount=11 (was 11), AvailableMemoryMB=10394 (was 10690)