2024-11-13 05:52:05,175 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 05:52:05,194 main DEBUG Took 0.017418 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 05:52:05,195 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 05:52:05,195 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 05:52:05,211 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 05:52:05,212 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,220 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 05:52:05,236 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,238 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,239 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,241 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,242 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,244 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,246 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,247 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,247 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,248 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,248 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,249 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,250 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 05:52:05,252 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,253 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 05:52:05,255 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 05:52:05,256 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 05:52:05,259 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 05:52:05,260 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 05:52:05,262 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 05:52:05,262 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 05:52:05,273 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 05:52:05,277 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 05:52:05,279 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 05:52:05,280 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 05:52:05,280 main DEBUG createAppenders(={Console}) 2024-11-13 05:52:05,281 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-13 05:52:05,281 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-13 05:52:05,282 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-13 05:52:05,283 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 05:52:05,283 main DEBUG OutputStream closed 2024-11-13 05:52:05,283 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 05:52:05,284 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 05:52:05,284 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-13 05:52:05,389 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 05:52:05,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 05:52:05,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 05:52:05,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 05:52:05,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 05:52:05,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 05:52:05,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 05:52:05,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 05:52:05,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 05:52:05,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 05:52:05,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 05:52:05,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 05:52:05,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 05:52:05,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 05:52:05,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 05:52:05,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 05:52:05,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 05:52:05,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 05:52:05,408 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 05:52:05,409 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-13 05:52:05,409 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 05:52:05,410 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-13T05:52:05,431 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-13 05:52:05,434 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 05:52:05,435 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T05:52:05,829 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39 2024-11-13T05:52:05,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=389, ProcessCount=11, AvailableMemoryMB=7710 2024-11-13T05:52:05,986 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T05:52:06,117 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:06,165 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/cluster_3b6c1f89-d8d3-bc40-c962-1dc4e6f25034, deleteOnExit=true 2024-11-13T05:52:06,360 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/cluster_3b6c1f89-d8d3-bc40-c962-1dc4e6f25034/zookeeper_0, clientPort=62789, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/cluster_3b6c1f89-d8d3-bc40-c962-1dc4e6f25034/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/cluster_3b6c1f89-d8d3-bc40-c962-1dc4e6f25034/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:52:06,374 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62789 2024-11-13T05:52:06,375 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:06,464 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 with version=8 2024-11-13T05:52:06,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase-staging 2024-11-13T05:52:06,576 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T05:52:06,922 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:06,931 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:06,933 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:06,944 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:06,944 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:06,945 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:07,138 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:52:07,249 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:52:07,315 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T05:52:07,329 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T05:52:07,333 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:07,376 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 40840 (auto-detected) 2024-11-13T05:52:07,377 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T05:52:07,406 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43295 2024-11-13T05:52:07,440 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43295 connecting to ZooKeeper ensemble=127.0.0.1:62789 2024-11-13T05:52:07,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432950x0, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:07,555 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43295-0x1013409cf820000 connected 2024-11-13T05:52:07,647 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:07,649 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:07,662 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:07,667 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2, hbase.cluster.distributed=false 2024-11-13T05:52:07,699 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:07,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-13T05:52:07,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43295 2024-11-13T05:52:07,707 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43295 2024-11-13T05:52:07,712 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-13T05:52:07,713 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43295 2024-11-13T05:52:07,822 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:07,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,825 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:07,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:07,829 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:52:07,834 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:07,835 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41281 2024-11-13T05:52:07,838 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41281 connecting to ZooKeeper ensemble=127.0.0.1:62789 2024-11-13T05:52:07,838 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:07,839 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:07,863 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412810x0, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:07,864 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:412810x0, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:07,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41281-0x1013409cf820001 connected 2024-11-13T05:52:07,870 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:52:07,882 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.cache 2024-11-13T05:52:07,888 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T05:52:07,889 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T05:52:07,891 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T05:52:07,892 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T05:52:07,894 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T05:52:07,901 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T05:52:07,903 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T05:52:07,903 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence is complete. 2024-11-13T05:52:07,910 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:52:07,913 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:52:07,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:07,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41281 2024-11-13T05:52:07,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41281 2024-11-13T05:52:07,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41281 2024-11-13T05:52:07,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41281 2024-11-13T05:52:07,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41281 2024-11-13T05:52:07,964 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:07,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,965 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,966 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:07,966 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:07,966 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:07,966 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:52:07,967 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:07,969 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45187 2024-11-13T05:52:07,972 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45187 connecting to ZooKeeper ensemble=127.0.0.1:62789 2024-11-13T05:52:07,973 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:07,974 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:08,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451870x0, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:08,007 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451870x0, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:08,008 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:52:08,008 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45187-0x1013409cf820002 connected 2024-11-13T05:52:08,011 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.cache 2024-11-13T05:52:08,011 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T05:52:08,011 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T05:52:08,012 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T05:52:08,014 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T05:52:08,014 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T05:52:08,015 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T05:52:08,015 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence is complete. 2024-11-13T05:52:08,019 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T05:52:08,022 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:52:08,024 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:52:08,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:08,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-13T05:52:08,034 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45187 2024-11-13T05:52:08,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45187 2024-11-13T05:52:08,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-13T05:52:08,041 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45187 2024-11-13T05:52:08,058 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:43295 2024-11-13T05:52:08,061 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:08,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,081 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:08,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,128 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:52:08,129 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,43295,1731477126718 from backup master directory 2024-11-13T05:52:08,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:08,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:08,144 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:08,144 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:08,147 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T05:52:08,149 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T05:52:08,160 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase.id] with ID: 638cece7-d778-426e-965e-6fcf7c209510 2024-11-13T05:52:08,161 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.tmp/hbase.id 2024-11-13T05:52:08,169 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase.id] 2024-11-13T05:52:08,222 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:08,227 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:52:08,253 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-11-13T05:52:08,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:08,332 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:08,338 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:52:08,397 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:08,404 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:08,512 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store 2024-11-13T05:52:08,565 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T05:52:08,569 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:08,573 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:52:08,573 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:08,574 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:08,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:52:08,576 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:08,577 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:08,577 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477128573Disabling compacts and flushes for region at 1731477128573Disabling writes for close at 1731477128576 (+3 ms)Writing region close event to WAL at 1731477128576Closed at 1731477128576 2024-11-13T05:52:08,578 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/.initializing 2024-11-13T05:52:08,579 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:08,602 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:08,622 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C43295%2C1731477126718, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,43295,1731477126718, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/oldWALs, maxLogs=10 2024-11-13T05:52:08,679 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,43295,1731477126718/bc386b7bf2fe%2C43295%2C1731477126718.1731477128630 2024-11-13T05:52:08,680 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:08,681 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:08,681 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:08,686 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,687 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,812 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:52:08,818 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,821 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:08,823 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:52:08,836 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,836 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:08,837 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:52:08,842 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:08,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:52:08,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:08,863 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:08,864 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,871 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,876 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,881 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,882 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,886 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:08,892 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:08,915 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:08,916 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73007325, jitterRate=0.08789391815662384}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:08,932 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731477128705Initializing all the Stores at 1731477128706 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477128706Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477128710 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477128711 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477128711Cleaning up temporary data from old regions at 1731477128882 (+171 ms)Region opened successfully at 1731477128932 (+50 ms) 2024-11-13T05:52:08,935 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:52:09,000 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ebde83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:09,050 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:52:09,073 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:52:09,074 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:52:09,078 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:52:09,079 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T05:52:09,094 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 14 msec 2024-11-13T05:52:09,094 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:52:09,147 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:52:09,162 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:52:09,213 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:52:09,216 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:52:09,218 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:52:09,231 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:52:09,234 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:52:09,241 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:52:09,248 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:52:09,251 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:52:09,265 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:52:09,285 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:52:09,296 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,325 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,43295,1731477126718, sessionid=0x1013409cf820000, setting cluster-up flag (Was=false) 2024-11-13T05:52:09,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,382 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:52:09,385 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:09,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:09,431 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:52:09,433 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:09,438 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:52:09,446 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(746): ClusterId : 638cece7-d778-426e-965e-6fcf7c209510 2024-11-13T05:52:09,446 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(746): ClusterId : 638cece7-d778-426e-965e-6fcf7c209510 2024-11-13T05:52:09,449 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:52:09,449 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:52:09,459 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:52:09,459 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:52:09,459 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:52:09,459 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:52:09,473 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:52:09,473 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:52:09,473 DEBUG [RS:1;bc386b7bf2fe:45187 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64086d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:09,473 DEBUG [RS:0;bc386b7bf2fe:41281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d737c91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:09,487 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bc386b7bf2fe:45187 2024-11-13T05:52:09,490 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:52:09,490 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:52:09,491 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:52:09,492 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:41281 2024-11-13T05:52:09,492 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:52:09,492 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:52:09,492 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:52:09,494 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,43295,1731477126718 with port=41281, startcode=1731477127775 2024-11-13T05:52:09,494 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,43295,1731477126718 with port=45187, startcode=1731477127963 2024-11-13T05:52:09,505 DEBUG [RS:0;bc386b7bf2fe:41281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:52:09,505 DEBUG [RS:1;bc386b7bf2fe:45187 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:52:09,519 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:09,531 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:52:09,539 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:52:09,545 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,43295,1731477126718 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:52:09,554 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:09,554 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:09,555 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:09,555 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:09,556 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:52:09,556 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,556 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:09,556 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,563 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477159563 2024-11-13T05:52:09,563 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:09,564 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:52:09,565 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:52:09,567 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:52:09,570 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:52:09,571 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:52:09,571 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:52:09,571 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:52:09,571 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,572 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:52:09,576 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41881, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:52:09,576 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34099, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:52:09,574 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,583 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:09,587 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:52:09,588 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:52:09,589 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:52:09,590 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-13T05:52:09,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:52:09,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:52:09,598 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477129596,5,FailOnTimeoutGroup] 2024-11-13T05:52:09,599 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:52:09,600 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:09,601 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477129599,5,FailOnTimeoutGroup] 2024-11-13T05:52:09,601 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,601 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:52:09,603 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,603 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,607 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:09,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T05:52:09,613 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:52:09,613 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,613 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:09,614 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T05:52:09,615 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T05:52:09,615 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-13T05:52:09,615 WARN [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T05:52:09,615 WARN [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-13T05:52:09,617 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:52:09,618 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:09,618 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T05:52:09,622 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:52:09,622 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:09,623 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T05:52:09,626 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:52:09,626 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:09,627 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:09,627 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:52:09,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:09,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:09,628 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:52:09,629 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:52:09,629 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:09,630 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:52:09,635 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:09,636 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61714661, jitterRate=-0.08037988841533661}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:09,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731477129608Initializing all the Stores at 1731477129608Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477129608Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477129608Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477129608Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477129608Cleaning up temporary data from old regions at 1731477129629 (+21 ms)Region opened successfully at 1731477129640 (+11 ms) 2024-11-13T05:52:09,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:52:09,640 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:52:09,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:52:09,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:52:09,641 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:52:09,642 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:09,643 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477129640Disabling compacts and flushes for region at 1731477129640Disabling writes for close at 1731477129641 (+1 ms)Writing region close event to WAL at 1731477129642 (+1 ms)Closed at 1731477129642 2024-11-13T05:52:09,645 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:09,645 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:52:09,652 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:52:09,659 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:09,662 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:52:09,716 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,43295,1731477126718 with port=41281, startcode=1731477127775 2024-11-13T05:52:09,716 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,43295,1731477126718 with port=45187, startcode=1731477127963 2024-11-13T05:52:09,719 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,720 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,726 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,727 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43295 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,727 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:09,727 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T05:52:09,727 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:52:09,730 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:09,731 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T05:52:09,731 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:52:09,763 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:09,796 DEBUG [RS:1;bc386b7bf2fe:45187 {}] zookeeper.ZKUtil(111): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,796 WARN [RS:1;bc386b7bf2fe:45187 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:09,797 INFO [RS:1;bc386b7bf2fe:45187 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:09,797 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,797 DEBUG [RS:0;bc386b7bf2fe:41281 {}] zookeeper.ZKUtil(111): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,798 WARN [RS:0;bc386b7bf2fe:41281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:09,798 INFO [RS:0;bc386b7bf2fe:41281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:09,799 DEBUG [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,45187,1731477127963] 2024-11-13T05:52:09,800 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,41281,1731477127775] 2024-11-13T05:52:09,813 WARN [bc386b7bf2fe:43295 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:52:09,827 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:52:09,827 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:52:09,843 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:52:09,843 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:52:09,849 INFO [RS:1;bc386b7bf2fe:45187 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:52:09,849 INFO [RS:0;bc386b7bf2fe:41281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:52:09,849 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,849 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,850 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:52:09,850 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:52:09,855 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:52:09,855 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:52:09,856 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,856 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:09,857 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,857 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,858 DEBUG [RS:0;bc386b7bf2fe:41281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:09,858 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:09,859 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:09,859 DEBUG [RS:1;bc386b7bf2fe:45187 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,860 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,41281,1731477127775-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,861 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45187,1731477127963-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:09,881 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:52:09,881 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:52:09,883 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,45187,1731477127963-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,883 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,41281,1731477127775-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,883 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,883 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,883 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.Replication(171): bc386b7bf2fe,45187,1731477127963 started 2024-11-13T05:52:09,883 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.Replication(171): bc386b7bf2fe,41281,1731477127775 started 2024-11-13T05:52:09,902 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,902 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:09,902 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,45187,1731477127963, RpcServer on bc386b7bf2fe/172.17.0.2:45187, sessionid=0x1013409cf820002 2024-11-13T05:52:09,902 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,41281,1731477127775, RpcServer on bc386b7bf2fe/172.17.0.2:41281, sessionid=0x1013409cf820001 2024-11-13T05:52:09,903 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:52:09,903 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:52:09,903 DEBUG [RS:1;bc386b7bf2fe:45187 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,903 DEBUG [RS:0;bc386b7bf2fe:41281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,903 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45187,1731477127963' 2024-11-13T05:52:09,903 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,41281,1731477127775' 2024-11-13T05:52:09,904 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:52:09,904 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:52:09,905 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:52:09,905 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:52:09,905 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:52:09,905 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:52:09,905 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:52:09,905 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:52:09,906 DEBUG [RS:1;bc386b7bf2fe:45187 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:09,906 DEBUG [RS:0;bc386b7bf2fe:41281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:09,906 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,45187,1731477127963' 2024-11-13T05:52:09,906 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,41281,1731477127775' 2024-11-13T05:52:09,906 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:52:09,906 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:52:09,906 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:52:09,906 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:52:09,907 DEBUG [RS:1;bc386b7bf2fe:45187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:52:09,907 DEBUG [RS:0;bc386b7bf2fe:41281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:52:09,907 INFO [RS:1;bc386b7bf2fe:45187 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:52:09,907 INFO [RS:0;bc386b7bf2fe:41281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:52:09,907 INFO [RS:1;bc386b7bf2fe:45187 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:52:09,907 INFO [RS:0;bc386b7bf2fe:41281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:52:10,015 INFO [RS:0;bc386b7bf2fe:41281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:10,015 INFO [RS:1;bc386b7bf2fe:45187 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:10,017 INFO [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C45187%2C1731477127963, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,45187,1731477127963, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:10,017 INFO [RS:0;bc386b7bf2fe:41281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C41281%2C1731477127775, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41281,1731477127775, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:10,039 INFO [RS:0;bc386b7bf2fe:41281 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41281,1731477127775/bc386b7bf2fe%2C41281%2C1731477127775.1731477130022 2024-11-13T05:52:10,039 INFO [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,45187,1731477127963/bc386b7bf2fe%2C45187%2C1731477127963.1731477130022 2024-11-13T05:52:10,039 DEBUG [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:10,040 DEBUG [RS:0;bc386b7bf2fe:41281 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:10,065 DEBUG [bc386b7bf2fe:43295 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T05:52:10,076 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:52:10,079 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:52:10,079 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:52:10,079 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:52:10,079 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:52:10,079 INFO [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:52:10,079 INFO [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:52:10,079 DEBUG [bc386b7bf2fe:43295 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:52:10,085 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:10,090 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45187,1731477127963, state=OPENING 2024-11-13T05:52:10,123 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:52:10,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:10,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:10,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:10,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,183 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:10,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45187,1731477127963}] 2024-11-13T05:52:10,368 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:52:10,371 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52095, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:52:10,380 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:52:10,380 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:10,381 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:52:10,383 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C45187%2C1731477127963.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,45187,1731477127963, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:10,403 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,45187,1731477127963/bc386b7bf2fe%2C45187%2C1731477127963.meta.1731477130385.meta 2024-11-13T05:52:10,404 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:10,404 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:10,406 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:52:10,408 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:52:10,414 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:52:10,417 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:52:10,418 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:10,418 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:52:10,418 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:52:10,420 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T05:52:10,421 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:52:10,421 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:10,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:10,421 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T05:52:10,422 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:52:10,422 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:10,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:10,422 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T05:52:10,423 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:52:10,423 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:10,423 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:10,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T05:52:10,424 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:52:10,424 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:10,425 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:10,425 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:52:10,426 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:10,427 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:10,428 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:52:10,428 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:52:10,429 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:10,430 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:52:10,431 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70639735, jitterRate=0.05261407792568207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:10,432 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:52:10,434 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477130419Writing region info on filesystem at 1731477130419Initializing all the Stores at 1731477130419Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477130419Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477130420 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477130420Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477130420Cleaning up temporary data from old regions at 1731477130428 (+8 ms)Running coprocessor post-open hooks at 1731477130432 (+4 ms)Region opened successfully at 1731477130434 (+2 ms) 2024-11-13T05:52:10,439 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731477130358 2024-11-13T05:52:10,448 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:52:10,449 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:52:10,450 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:10,451 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,45187,1731477127963, state=OPEN 2024-11-13T05:52:10,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:10,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:10,488 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:10,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,488 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:10,488 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:10,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:52:10,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,45187,1731477127963 in 303 msec 2024-11-13T05:52:10,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:52:10,495 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 842 msec 2024-11-13T05:52:10,496 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:10,496 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:52:10,518 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:10,519 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45187,1731477127963, seqNum=-1] 2024-11-13T05:52:10,541 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:10,543 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56897, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:10,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0890 sec 2024-11-13T05:52:10,561 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731477130561, completionTime=-1 2024-11-13T05:52:10,563 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T05:52:10,564 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:52:10,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:52:10,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477190595 2024-11-13T05:52:10,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477250595 2024-11-13T05:52:10,595 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-13T05:52:10,598 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T05:52:10,606 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,606 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,606 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,608 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:43295, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,609 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,609 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,615 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:52:10,641 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.496sec 2024-11-13T05:52:10,643 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:52:10,644 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:52:10,646 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:52:10,646 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:52:10,646 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:52:10,647 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:10,648 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:52:10,649 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:52:10,650 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:52:10,651 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,43295,1731477126718-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:10,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:10,661 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T05:52:10,661 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T05:52:10,666 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,43295,-1 for getting cluster id 2024-11-13T05:52:10,669 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:10,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '638cece7-d778-426e-965e-6fcf7c209510' 2024-11-13T05:52:10,680 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:10,681 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "638cece7-d778-426e-965e-6fcf7c209510" 2024-11-13T05:52:10,681 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a490df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:10,681 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,43295,-1] 2024-11-13T05:52:10,683 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:10,685 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:10,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:10,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:10,690 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:10,698 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,45187,1731477127963, seqNum=-1] 2024-11-13T05:52:10,698 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:10,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:10,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:10,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:52:10,750 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:10,752 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a9c1634 2024-11-13T05:52:10,753 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:52:10,756 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:52:10,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731477130729', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 2024-11-13T05:52:10,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:52:10,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731477130729" procId is: 4 2024-11-13T05:52:10,776 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:10,777 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:52:10,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:10,791 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b2a753c79c92c050d31b61f32386dd9e, NAME => 'testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731477130729', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1722): Closing b2a753c79c92c050d31b61f32386dd9e, disabling compactions & flushes 2024-11-13T05:52:10,797 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. after waiting 0 ms 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:10,797 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:10,797 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731477130729-pool-0 {}] regionserver.HRegion(1676): Region close journal for b2a753c79c92c050d31b61f32386dd9e: Waiting for close lock at 1731477130797Disabling compacts and flushes for region at 1731477130797Disabling writes for close at 1731477130797Writing region close event to WAL at 1731477130797Closed at 1731477130797 2024-11-13T05:52:10,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:52:10,804 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731477130799"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477130799"}]},"ts":"1731477130799"} 2024-11-13T05:52:10,808 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:52:10,810 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:52:10,812 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731477130729","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477130810"}]},"ts":"1731477130810"} 2024-11-13T05:52:10,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731477130729, state=ENABLING in hbase:meta 2024-11-13T05:52:10,816 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:52:10,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:52:10,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:52:10,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:52:10,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:52:10,817 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:52:10,817 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:52:10,817 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:52:10,818 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, ASSIGN}] 2024-11-13T05:52:10,820 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, ASSIGN 2024-11-13T05:52:10,821 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,41281,1731477127775; forceNewPlan=false, retain=false 2024-11-13T05:52:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:10,975 INFO [bc386b7bf2fe:43295 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:52:10,976 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=OPENING, regionLocation=bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:10,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, ASSIGN because future has completed 2024-11-13T05:52:10,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775}] 2024-11-13T05:52:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:11,138 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:52:11,140 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:52:11,142 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:11,143 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b2a753c79c92c050d31b61f32386dd9e, NAME => 'testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:11,143 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731477130729 b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,144 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:11,144 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,144 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,145 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,148 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a753c79c92c050d31b61f32386dd9e columnFamilyName family 2024-11-13T05:52:11,149 DEBUG [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:11,149 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(327): Store=b2a753c79c92c050d31b61f32386dd9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:11,149 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,150 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,150 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,151 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,151 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,151 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,157 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:11,157 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b2a753c79c92c050d31b61f32386dd9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61868917, jitterRate=-0.07808129489421844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:52:11,158 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:11,158 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b2a753c79c92c050d31b61f32386dd9e: Running coprocessor pre-open hook at 1731477131144Writing region info on filesystem at 1731477131144Initializing all the Stores at 1731477131144Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477131144Cleaning up temporary data from old regions at 1731477131151 (+7 ms)Running coprocessor post-open hooks at 1731477131158 (+7 ms)Region opened successfully at 1731477131158 2024-11-13T05:52:11,160 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e., pid=6, masterSystemTime=1731477131137 2024-11-13T05:52:11,163 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:11,163 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:11,163 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:11,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775 because future has completed 2024-11-13T05:52:11,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:52:11,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775 in 184 msec 2024-11-13T05:52:11,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:52:11,173 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, ASSIGN in 353 msec 2024-11-13T05:52:11,174 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:52:11,174 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731477130729","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477131174"}]},"ts":"1731477131174"} 2024-11-13T05:52:11,177 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731477130729, state=ENABLED in hbase:meta 2024-11-13T05:52:11,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:52:11,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731477130729 in 411 msec 2024-11-13T05:52:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:11,415 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731477130729 completed 2024-11-13T05:52:11,415 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731477130729 get assigned. Timeout = 60000ms 2024-11-13T05:52:11,419 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:52:11,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731477130729 assigned to meta. Checking AM states. 2024-11-13T05:52:11,426 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:52:11,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731477130729 assigned. 2024-11-13T05:52:11,435 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731477130729', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e., hostname=bc386b7bf2fe,41281,1731477127775, seqNum=2] 2024-11-13T05:52:11,436 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:11,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:11,448 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b2a753c79c92c050d31b61f32386dd9e 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T05:52:11,470 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:11,505 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/.tmp/family/42719a1681aa4147bff882f5c36adaaa is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T05:52:11,510 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/.tmp/family/42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:11,542 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/.tmp/family/42719a1681aa4147bff882f5c36adaaa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:11,544 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, delay=911 ms 2024-11-13T05:52:11,545 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T05:52:11,551 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for b2a753c79c92c050d31b61f32386dd9e in 101ms, sequenceid=6, compaction requested=false 2024-11-13T05:52:11,551 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731477130729' 2024-11-13T05:52:11,553 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b2a753c79c92c050d31b61f32386dd9e: 2024-11-13T05:52:12,461 DEBUG [hfile-prefetch-1731477131544 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 42719a1681aa4147bff882f5c36adaaa_0. 2024-11-13T05:52:12,466 DEBUG [hfile-prefetch-1731477131544 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 42719a1681aa4147bff882f5c36adaaa_117. 2024-11-13T05:52:12,467 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, with total blocks 2, and data blocks 1 2024-11-13T05:52:12,467 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T05:52:12,468 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T05:52:12,468 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa. Will sleep 100ms and try the verification again. 2024-11-13T05:52:12,568 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, with total blocks 2, and data blocks 1 2024-11-13T05:52:12,568 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-13T05:52:12,569 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2294): found block for file 42719a1681aa4147bff882f5c36adaaa in the backing map. Acquiring read lock for offset 0 2024-11-13T05:52:12,569 DEBUG [hfile-prefetch-1731477131544 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa has now been fully cached. 2024-11-13T05:52:12,569 DEBUG [hfile-prefetch-1731477131544 {}] hfile.PrefetchExecutor(121): Prefetch completed for 42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:12,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=b2a753c79c92c050d31b61f32386dd9e, source=bc386b7bf2fe,41281,1731477127775, destination=bc386b7bf2fe,45187,1731477127963, warming up region on bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:12,573 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=b2a753c79c92c050d31b61f32386dd9e, source=bc386b7bf2fe,41281,1731477127775, destination=bc386b7bf2fe,45187,1731477127963, running balancer 2024-11-13T05:52:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE 2024-11-13T05:52:12,575 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE 2024-11-13T05:52:12,575 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(7855): Warmup {ENCODED => b2a753c79c92c050d31b61f32386dd9e, NAME => 'testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:12,576 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:12,577 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=CLOSING, regionLocation=bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:12,577 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a753c79c92c050d31b61f32386dd9e columnFamilyName family 2024-11-13T05:52:12,577 DEBUG [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:12,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE because future has completed 2024-11-13T05:52:12,581 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-13T05:52:12,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775}] 2024-11-13T05:52:12,582 DEBUG [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:12,582 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(327): Store=b2a753c79c92c050d31b61f32386dd9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1722): Closing b2a753c79c92c050d31b61f32386dd9e, disabling compactions & flushes 2024-11-13T05:52:12,582 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. after waiting 0 ms 2024-11-13T05:52:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45187 {}] regionserver.HRegion(1676): Region close journal for b2a753c79c92c050d31b61f32386dd9e: Waiting for close lock at 1731477132582Disabling compacts and flushes for region at 1731477132582Disabling writes for close at 1731477132583 (+1 ms)Writing region close event to WAL at 1731477132584 (+1 ms)Closed at 1731477132584 2024-11-13T05:52:12,596 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T05:52:12,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T05:52:12,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:12,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-13T05:52:12,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-13T05:52:12,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731477132596 2024-11-13T05:52:12,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T05:52:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43295 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T05:52:12,744 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:12,744 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-13T05:52:12,745 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing b2a753c79c92c050d31b61f32386dd9e, disabling compactions & flushes 2024-11-13T05:52:12,746 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,746 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,746 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. after waiting 0 ms 2024-11-13T05:52:12,746 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,749 DEBUG [StoreFileCloser-b2a753c79c92c050d31b61f32386dd9e-family-1 {}] bucket.BucketCache(769): Removing file 42719a1681aa4147bff882f5c36adaaa for region b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:12,750 DEBUG [StoreFileCloser-b2a753c79c92c050d31b61f32386dd9e-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 42719a1681aa4147bff882f5c36adaaa, starting offset: 0, end offset: 9223372036854775807 2024-11-13T05:52:12,752 DEBUG [StoreFileCloser-b2a753c79c92c050d31b61f32386dd9e-family-1 {}] bucket.BucketCache(857): removed key 42719a1681aa4147bff882f5c36adaaa_0 from back map with offset lock 2094080 in the evict process 2024-11-13T05:52:12,758 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T05:52:12,760 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:12,760 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for b2a753c79c92c050d31b61f32386dd9e: Waiting for close lock at 1731477132745Running coprocessor pre-close hooks at 1731477132745Disabling compacts and flushes for region at 1731477132745Disabling writes for close at 1731477132746 (+1 ms)Writing region close event to WAL at 1731477132753 (+7 ms)Running coprocessor post-close hooks at 1731477132758 (+5 ms)Closed at 1731477132760 (+2 ms) 2024-11-13T05:52:12,760 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding b2a753c79c92c050d31b61f32386dd9e move to bc386b7bf2fe,45187,1731477127963 record at close sequenceid=6 2024-11-13T05:52:12,765 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:12,765 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=CLOSED 2024-11-13T05:52:12,769 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775 because future has completed 2024-11-13T05:52:12,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:52:12,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,41281,1731477127775 in 189 msec 2024-11-13T05:52:12,774 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE; state=CLOSED, location=bc386b7bf2fe,45187,1731477127963; forceNewPlan=false, retain=false 2024-11-13T05:52:12,815 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T05:52:12,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T05:52:12,816 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:12,816 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:12,818 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T05:52:12,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T05:52:12,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:12,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:12,819 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:12,819 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477132815 2024-11-13T05:52:12,819 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T05:52:12,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:12,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477132818 2024-11-13T05:52:12,821 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T05:52:12,925 INFO [bc386b7bf2fe:43295 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:52:12,926 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=OPENING, regionLocation=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:12,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE because future has completed 2024-11-13T05:52:12,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,45187,1731477127963}] 2024-11-13T05:52:13,092 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,093 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => b2a753c79c92c050d31b61f32386dd9e, NAME => 'testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:13,094 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731477130729 b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,094 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:13,094 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,095 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,097 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,099 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2a753c79c92c050d31b61f32386dd9e columnFamilyName family 2024-11-13T05:52:13,099 DEBUG [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:13,101 DEBUG [StoreFileOpener-b2a753c79c92c050d31b61f32386dd9e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa, delay=919 ms 2024-11-13T05:52:13,103 DEBUG [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:13,103 INFO [StoreOpener-b2a753c79c92c050d31b61f32386dd9e-1 {}] regionserver.HStore(327): Store=b2a753c79c92c050d31b61f32386dd9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:13,103 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,104 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,104 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,104 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,104 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,108 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,109 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened b2a753c79c92c050d31b61f32386dd9e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71390840, jitterRate=0.06380641460418701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:52:13,109 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,110 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for b2a753c79c92c050d31b61f32386dd9e: Running coprocessor pre-open hook at 1731477133095Writing region info on filesystem at 1731477133095Initializing all the Stores at 1731477133096 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477133096Cleaning up temporary data from old regions at 1731477133104 (+8 ms)Running coprocessor post-open hooks at 1731477133109 (+5 ms)Region opened successfully at 1731477133110 (+1 ms) 2024-11-13T05:52:13,110 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e., pid=9, masterSystemTime=1731477133087 2024-11-13T05:52:13,114 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,114 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,114 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=b2a753c79c92c050d31b61f32386dd9e, regionState=OPEN, openSeqNum=10, regionLocation=bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:13,118 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,45187,1731477127963 because future has completed 2024-11-13T05:52:13,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T05:52:13,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure b2a753c79c92c050d31b61f32386dd9e, server=bc386b7bf2fe,45187,1731477127963 in 186 msec 2024-11-13T05:52:13,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731477130729, region=b2a753c79c92c050d31b61f32386dd9e, REOPEN/MOVE in 548 msec 2024-11-13T05:52:13,127 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:52:13,127 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:13,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:13,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,133 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:13,133 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:52:13,134 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-13T05:52:13,134 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,43295,1731477126718 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:13,164 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:52:13,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:13,165 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:13,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:13,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,166 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,41281,1731477127775' ***** 2024-11-13T05:52:13,166 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:52:13,167 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,45187,1731477127963' ***** 2024-11-13T05:52:13,167 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:52:13,167 INFO [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T05:52:13,167 INFO [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T05:52:13,167 INFO [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence 2024-11-13T05:52:13,167 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T05:52:13,167 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T05:52:13,167 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T05:52:13,167 INFO [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/bucket.persistence 2024-11-13T05:52:13,168 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T05:52:13,167 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T05:52:13,168 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T05:52:13,168 INFO [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T05:52:13,168 INFO [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T05:52:13,168 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T05:52:13,168 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T05:52:13,178 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1488): Thread RS:0;bc386b7bf2fe:41281 started persisting bucket cache to file 2024-11-13T05:52:13,178 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1488): Thread RS:1;bc386b7bf2fe:45187 started persisting bucket cache to file 2024-11-13T05:52:13,178 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:13,178 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:13,178 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:13,178 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:13,181 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:13,181 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477133178 2024-11-13T05:52:13,181 ERROR [RS:0;bc386b7bf2fe:41281 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1775) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:13,182 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:13,182 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477133178 2024-11-13T05:52:13,182 DEBUG [RS:0;bc386b7bf2fe:41281 {}] bucket.BucketCache(1510): Thread RS:0;bc386b7bf2fe:41281 finished persisting bucket cache to file, renaming 2024-11-13T05:52:13,182 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:52:13,182 INFO [RS:0;bc386b7bf2fe:41281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:52:13,182 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:52:13,182 INFO [RS:0;bc386b7bf2fe:41281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:52:13,183 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:13,183 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:13,183 INFO [RS:0;bc386b7bf2fe:41281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:41281. 2024-11-13T05:52:13,183 DEBUG [RS:0;bc386b7bf2fe:41281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:13,183 DEBUG [RS:0;bc386b7bf2fe:41281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,183 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,41281,1731477127775; all regions closed. 2024-11-13T05:52:13,187 DEBUG [RS:0;bc386b7bf2fe:41281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:13,188 INFO [RS:0;bc386b7bf2fe:41281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C41281%2C1731477127775:(num 1731477130022) 2024-11-13T05:52:13,188 DEBUG [RS:0;bc386b7bf2fe:41281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,188 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:13,188 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:13,188 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:13,189 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:52:13,189 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:13,189 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:52:13,189 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:52:13,189 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:13,189 INFO [RS:0;bc386b7bf2fe:41281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41281 2024-11-13T05:52:13,196 DEBUG [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1510): Thread RS:1;bc386b7bf2fe:45187 finished persisting bucket cache to file, renaming 2024-11-13T05:52:13,196 WARN [RS:1;bc386b7bf2fe:45187 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:52:13,197 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(3091): Received CLOSE for b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:13,197 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bc386b7bf2fe:45187. 2024-11-13T05:52:13,198 DEBUG [RS:1;bc386b7bf2fe:45187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:13,198 DEBUG [RS:1;bc386b7bf2fe:45187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:52:13,198 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:52:13,198 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b2a753c79c92c050d31b61f32386dd9e, disabling compactions & flushes 2024-11-13T05:52:13,198 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,198 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,198 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. after waiting 0 ms 2024-11-13T05:52:13,198 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:52:13,198 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,199 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:13,199 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1325): Online Regions={b2a753c79c92c050d31b61f32386dd9e=testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e., 1588230740=hbase:meta,,1.1588230740} 2024-11-13T05:52:13,199 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:52:13,199 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:52:13,199 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:52:13,199 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:52:13,199 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:52:13,199 DEBUG [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, b2a753c79c92c050d31b61f32386dd9e 2024-11-13T05:52:13,199 DEBUG [StoreFileCloser-b2a753c79c92c050d31b61f32386dd9e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/family/42719a1681aa4147bff882f5c36adaaa 2024-11-13T05:52:13,199 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-13T05:52:13,204 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnRegionMove1731477130729/b2a753c79c92c050d31b61f32386dd9e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T05:52:13,205 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,205 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b2a753c79c92c050d31b61f32386dd9e: Waiting for close lock at 1731477133198Running coprocessor pre-close hooks at 1731477133198Disabling compacts and flushes for region at 1731477133198Disabling writes for close at 1731477133198Writing region close event to WAL at 1731477133200 (+2 ms)Running coprocessor post-close hooks at 1731477133205 (+5 ms)Closed at 1731477133205 2024-11-13T05:52:13,205 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e. 2024-11-13T05:52:13,212 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8458ba5d68974a03b2a1a512aeba2746 2024-11-13T05:52:13,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:13,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,41281,1731477127775 2024-11-13T05:52:13,222 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:13,230 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/8458ba5d68974a03b2a1a512aeba2746 is 205, key is testBlockEvictionOnRegionMove1731477130729,,1731477130758.b2a753c79c92c050d31b61f32386dd9e./info:regioninfo/1731477133114/Put/seqid=0 2024-11-13T05:52:13,231 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,41281,1731477127775] 2024-11-13T05:52:13,231 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/8458ba5d68974a03b2a1a512aeba2746 2024-11-13T05:52:13,239 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,41281,1731477127775 already deleted, retry=false 2024-11-13T05:52:13,239 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,41281,1731477127775 expired; onlineServers=1 2024-11-13T05:52:13,240 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3938951114204773856c5bc1be6375b8 2024-11-13T05:52:13,253 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/3938951114204773856c5bc1be6375b8 is 43, key is default/ns:d/1731477130548/Put/seqid=0 2024-11-13T05:52:13,253 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/3938951114204773856c5bc1be6375b8 2024-11-13T05:52:13,262 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07ae04078609416ea85030686604ef8e 2024-11-13T05:52:13,264 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:13,264 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:13,275 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/07ae04078609416ea85030686604ef8e is 78, key is testBlockEvictionOnRegionMove1731477130729/table:state/1731477131174/Put/seqid=0 2024-11-13T05:52:13,276 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/07ae04078609416ea85030686604ef8e 2024-11-13T05:52:13,279 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/8458ba5d68974a03b2a1a512aeba2746 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/8458ba5d68974a03b2a1a512aeba2746 2024-11-13T05:52:13,280 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/8458ba5d68974a03b2a1a512aeba2746, delay=934 ms 2024-11-13T05:52:13,281 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/8458ba5d68974a03b2a1a512aeba2746, entries=15, sequenceid=15, filesize=7.6 K 2024-11-13T05:52:13,282 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/3938951114204773856c5bc1be6375b8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/3938951114204773856c5bc1be6375b8 2024-11-13T05:52:13,284 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/3938951114204773856c5bc1be6375b8, delay=906 ms 2024-11-13T05:52:13,286 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/3938951114204773856c5bc1be6375b8, entries=2, sequenceid=15, filesize=5.0 K 2024-11-13T05:52:13,286 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/07ae04078609416ea85030686604ef8e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/07ae04078609416ea85030686604ef8e 2024-11-13T05:52:13,287 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/07ae04078609416ea85030686604ef8e, delay=939 ms 2024-11-13T05:52:13,288 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/07ae04078609416ea85030686604ef8e, entries=2, sequenceid=15, filesize=5.3 K 2024-11-13T05:52:13,289 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-11-13T05:52:13,290 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T05:52:13,291 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/8458ba5d68974a03b2a1a512aeba2746 2024-11-13T05:52:13,291 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/3938951114204773856c5bc1be6375b8 2024-11-13T05:52:13,292 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/07ae04078609416ea85030686604ef8e 2024-11-13T05:52:13,296 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T05:52:13,296 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:52:13,297 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:13,297 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477133198Running coprocessor pre-close hooks at 1731477133199 (+1 ms)Disabling compacts and flushes for region at 1731477133199Disabling writes for close at 1731477133199Obtaining lock to block concurrent updates at 1731477133199Preparing flush snapshotting stores in 1588230740 at 1731477133199Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731477133200 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731477133200Flushing 1588230740/info: creating writer at 1731477133200Flushing 1588230740/info: appending metadata at 1731477133229 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731477133229Flushing 1588230740/ns: creating writer at 1731477133234 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731477133252 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731477133252Flushing 1588230740/table: creating writer at 1731477133256 (+4 ms)Flushing 1588230740/table: appending metadata at 1731477133275 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731477133275Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@272a7180: reopening flushed file at 1731477133278 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d567993: reopening flushed file at 1731477133282 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7288b7e8: reopening flushed file at 1731477133286 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1731477133289 (+3 ms)Writing region close event to WAL at 1731477133292 (+3 ms)Running coprocessor post-close hooks at 1731477133296 (+4 ms)Closed at 1731477133297 (+1 ms) 2024-11-13T05:52:13,297 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41281-0x1013409cf820001, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,332 INFO [RS:0;bc386b7bf2fe:41281 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:13,332 INFO [RS:0;bc386b7bf2fe:41281 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,41281,1731477127775; zookeeper connection closed. 2024-11-13T05:52:13,332 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c2290c6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c2290c6 2024-11-13T05:52:13,399 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,45187,1731477127963; all regions closed. 2024-11-13T05:52:13,406 DEBUG [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:13,406 INFO [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45187%2C1731477127963.meta:.meta(num 1731477130385) 2024-11-13T05:52:13,411 DEBUG [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:13,411 INFO [RS:1;bc386b7bf2fe:45187 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C45187%2C1731477127963:(num 1731477130022) 2024-11-13T05:52:13,411 DEBUG [RS:1;bc386b7bf2fe:45187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:13,411 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:13,411 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:13,412 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:13,412 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:13,412 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:13,412 INFO [RS:1;bc386b7bf2fe:45187 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45187 2024-11-13T05:52:13,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:13,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,45187,1731477127963 2024-11-13T05:52:13,446 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:13,455 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,45187,1731477127963] 2024-11-13T05:52:13,464 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,45187,1731477127963 already deleted, retry=false 2024-11-13T05:52:13,464 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,45187,1731477127963 expired; onlineServers=0 2024-11-13T05:52:13,464 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,43295,1731477126718' ***** 2024-11-13T05:52:13,464 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:52:13,464 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:13,464 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:13,464 DEBUG [M:0;bc386b7bf2fe:43295 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:52:13,465 DEBUG [M:0;bc386b7bf2fe:43295 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:52:13,465 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:52:13,465 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477129596 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477129596,5,FailOnTimeoutGroup] 2024-11-13T05:52:13,465 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477129599 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477129599,5,FailOnTimeoutGroup] 2024-11-13T05:52:13,465 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:13,465 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:13,465 DEBUG [M:0;bc386b7bf2fe:43295 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:52:13,465 INFO [M:0;bc386b7bf2fe:43295 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:52:13,465 INFO [M:0;bc386b7bf2fe:43295 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:52:13,466 INFO [M:0;bc386b7bf2fe:43295 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:52:13,466 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:52:13,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:13,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:13,472 DEBUG [M:0;bc386b7bf2fe:43295 {}] zookeeper.ZKUtil(347): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:52:13,472 WARN [M:0;bc386b7bf2fe:43295 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:52:13,473 INFO [M:0;bc386b7bf2fe:43295 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.lastflushedseqids 2024-11-13T05:52:13,483 INFO [M:0;bc386b7bf2fe:43295 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:52:13,484 INFO [M:0;bc386b7bf2fe:43295 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:52:13,484 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:52:13,484 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:13,484 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:13,484 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:52:13,484 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:13,484 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-13T05:52:13,493 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed656156404a4e19a112677366d82828 2024-11-13T05:52:13,507 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed656156404a4e19a112677366d82828 is 82, key is hbase:meta,,1/info:regioninfo/1731477130449/Put/seqid=0 2024-11-13T05:52:13,508 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed656156404a4e19a112677366d82828 2024-11-13T05:52:13,515 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 277580b4837a402288ceb6a1225b3c55 2024-11-13T05:52:13,529 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/277580b4837a402288ceb6a1225b3c55 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731477131179/Put/seqid=0 2024-11-13T05:52:13,529 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/277580b4837a402288ceb6a1225b3c55 2024-11-13T05:52:13,537 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4af1201fa0064393ba3e988e081f9000 2024-11-13T05:52:13,548 DEBUG [M:0;bc386b7bf2fe:43295 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4af1201fa0064393ba3e988e081f9000 is 69, key is bc386b7bf2fe,41281,1731477127775/rs:state/1731477129727/Put/seqid=0 2024-11-13T05:52:13,549 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4af1201fa0064393ba3e988e081f9000 2024-11-13T05:52:13,551 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed656156404a4e19a112677366d82828 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ed656156404a4e19a112677366d82828 2024-11-13T05:52:13,552 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ed656156404a4e19a112677366d82828, entries=8, sequenceid=84, filesize=5.5 K 2024-11-13T05:52:13,553 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/277580b4837a402288ceb6a1225b3c55 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/277580b4837a402288ceb6a1225b3c55 2024-11-13T05:52:13,554 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/277580b4837a402288ceb6a1225b3c55, entries=9, sequenceid=84, filesize=6.5 K 2024-11-13T05:52:13,555 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4af1201fa0064393ba3e988e081f9000 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4af1201fa0064393ba3e988e081f9000 2024-11-13T05:52:13,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,556 INFO [RS:1;bc386b7bf2fe:45187 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:13,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45187-0x1013409cf820002, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,556 INFO [RS:1;bc386b7bf2fe:45187 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,45187,1731477127963; zookeeper connection closed. 2024-11-13T05:52:13,556 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1adced14 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1adced14 2024-11-13T05:52:13,556 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4af1201fa0064393ba3e988e081f9000, entries=2, sequenceid=84, filesize=5.1 K 2024-11-13T05:52:13,556 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-13T05:52:13,557 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false 2024-11-13T05:52:13,559 INFO [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:13,559 DEBUG [M:0;bc386b7bf2fe:43295 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477133484Disabling compacts and flushes for region at 1731477133484Disabling writes for close at 1731477133484Obtaining lock to block concurrent updates at 1731477133484Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731477133484Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731477133485 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731477133485Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731477133485Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731477133507 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731477133507Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731477133510 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731477133528 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731477133528Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731477133531 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731477133548 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731477133548Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1731477133550 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1731477133553 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e78092: reopening flushed file at 1731477133554 (+1 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false at 1731477133557 (+3 ms)Writing region close event to WAL at 1731477133559 (+2 ms)Closed at 1731477133559 2024-11-13T05:52:13,561 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:13,561 INFO [M:0;bc386b7bf2fe:43295 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:52:13,561 INFO [M:0;bc386b7bf2fe:43295 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43295 2024-11-13T05:52:13,561 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,696 INFO [M:0;bc386b7bf2fe:43295 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:13,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43295-0x1013409cf820000, quorum=127.0.0.1:62789, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:13,703 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:52:13,724 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:52:13,724 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731477131544 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bc386b7bf2fe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731477133280 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731477133284 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bc386b7bf2fe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/bc386b7bf2fe:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731477133101 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/bc386b7bf2fe:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=366 (was 389), ProcessCount=11 (was 11), AvailableMemoryMB=7346 (was 7710) 2024-11-13T05:52:13,737 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=366, ProcessCount=11, AvailableMemoryMB=7347 2024-11-13T05:52:13,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/hadoop-log-dir so I do NOT create it in target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568 2024-11-13T05:52:13,737 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/hadoop-tmp-dir so I do NOT create it in target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568 2024-11-13T05:52:13,737 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,740 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/cluster_2a30bcb5-ae55-654d-4f1a-2f878bf2bb57, deleteOnExit=true 2024-11-13T05:52:13,743 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/cluster_2a30bcb5-ae55-654d-4f1a-2f878bf2bb57/zookeeper_0, clientPort=55047, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/cluster_2a30bcb5-ae55-654d-4f1a-2f878bf2bb57/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/cluster_2a30bcb5-ae55-654d-4f1a-2f878bf2bb57/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T05:52:13,744 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55047 2024-11-13T05:52:13,755 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 with version=8 2024-11-13T05:52:13,755 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase-staging 2024-11-13T05:52:13,758 INFO [Time-limited test {}] client.ConnectionUtils(128): master/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:13,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,758 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,759 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:13,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,759 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:13,759 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T05:52:13,759 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:13,760 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42689 2024-11-13T05:52:13,762 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42689 connecting to ZooKeeper ensemble=127.0.0.1:55047 2024-11-13T05:52:13,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:426890x0, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:13,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42689-0x1013409ec730000 connected 2024-11-13T05:52:13,874 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,875 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,878 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2, hbase.cluster.distributed=false 2024-11-13T05:52:13,882 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:13,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-13T05:52:13,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42689 2024-11-13T05:52:13,883 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42689 2024-11-13T05:52:13,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-13T05:52:13,884 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42689 2024-11-13T05:52:13,907 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:13,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,907 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:13,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,907 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:13,908 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:52:13,908 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:13,908 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41307 2024-11-13T05:52:13,910 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41307 connecting to ZooKeeper ensemble=127.0.0.1:55047 2024-11-13T05:52:13,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,910 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413070x0, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:13,922 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41307-0x1013409ec730001 connected 2024-11-13T05:52:13,922 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,922 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:52:13,923 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache 2024-11-13T05:52:13,923 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T05:52:13,923 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T05:52:13,923 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T05:52:13,924 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T05:52:13,924 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T05:52:13,924 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T05:52:13,924 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:52:13,924 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T05:52:13,924 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence is complete. 2024-11-13T05:52:13,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:52:13,926 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:13,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41307 2024-11-13T05:52:13,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41307 2024-11-13T05:52:13,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41307 2024-11-13T05:52:13,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41307 2024-11-13T05:52:13,933 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41307 2024-11-13T05:52:13,948 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:52:13,948 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:13,949 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35173 2024-11-13T05:52:13,951 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35173 connecting to ZooKeeper ensemble=127.0.0.1:55047 2024-11-13T05:52:13,951 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,951 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:13,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:351730x0, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:13,963 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35173-0x1013409ec730002 connected 2024-11-13T05:52:13,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:13,964 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:52:13,964 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache 2024-11-13T05:52:13,965 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T05:52:13,965 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T05:52:13,965 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T05:52:13,965 INFO [Thread-121 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T05:52:13,965 WARN [Thread-121 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-13T05:52:13,965 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T05:52:13,966 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:52:13,966 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T05:52:13,966 INFO [Thread-121 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence is complete. 2024-11-13T05:52:13,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T05:52:13,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:13,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35173 2024-11-13T05:52:13,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35173 2024-11-13T05:52:13,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35173 2024-11-13T05:52:13,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35173 2024-11-13T05:52:13,974 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35173 2024-11-13T05:52:13,990 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;bc386b7bf2fe:42689 2024-11-13T05:52:13,990 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:13,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:13,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:13,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:14,000 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:14,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:14,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,008 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:52:14,008 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/bc386b7bf2fe,42689,1731477133757 from backup master directory 2024-11-13T05:52:14,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:14,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:14,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T05:52:14,015 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:14,015 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,017 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase.id] with ID: 16daaa54-bfac-4105-9f17-c066e86323d8 2024-11-13T05:52:14,017 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.tmp/hbase.id 2024-11-13T05:52:14,020 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/hbase.id] 2024-11-13T05:52:14,050 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:14,051 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T05:52:14,054 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-13T05:52:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,070 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:14,071 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T05:52:14,071 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:14,095 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:52:14,104 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:14,104 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:14,104 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477134104Disabling compacts and flushes for region at 1731477134104Disabling writes for close at 1731477134104Writing region close event to WAL at 1731477134104Closed at 1731477134104 2024-11-13T05:52:14,105 WARN [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/.initializing 2024-11-13T05:52:14,105 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,111 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:14,115 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=bc386b7bf2fe%2C42689%2C1731477133757, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,42689,1731477133757, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/oldWALs, maxLogs=10 2024-11-13T05:52:14,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/WALs/bc386b7bf2fe,42689,1731477133757/bc386b7bf2fe%2C42689%2C1731477133757.1731477134115 2024-11-13T05:52:14,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:14,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:14,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:14,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,135 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T05:52:14,141 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:14,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T05:52:14,144 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:14,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,148 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T05:52:14,148 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:14,149 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T05:52:14,152 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:14,153 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,153 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,153 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,154 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,154 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,154 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:14,155 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T05:52:14,160 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:14,160 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61530032, jitterRate=-0.08313107490539551}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:14,162 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731477134136Initializing all the Stores at 1731477134136Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477134136Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477134136Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477134136Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477134136Cleaning up temporary data from old regions at 1731477134154 (+18 ms)Region opened successfully at 1731477134161 (+7 ms) 2024-11-13T05:52:14,162 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T05:52:14,166 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc9a941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:14,167 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T05:52:14,167 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T05:52:14,167 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T05:52:14,167 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T05:52:14,168 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T05:52:14,168 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-13T05:52:14,168 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T05:52:14,170 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T05:52:14,171 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T05:52:14,198 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T05:52:14,199 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T05:52:14,200 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T05:52:14,206 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T05:52:14,207 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T05:52:14,208 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T05:52:14,215 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T05:52:14,216 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T05:52:14,223 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T05:52:14,226 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T05:52:14,231 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,241 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=bc386b7bf2fe,42689,1731477133757, sessionid=0x1013409ec730000, setting cluster-up flag (Was=false) 2024-11-13T05:52:14,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,282 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T05:52:14,285 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,331 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T05:52:14,333 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:14,334 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T05:52:14,335 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:14,335 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T05:52:14,335 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T05:52:14,336 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: bc386b7bf2fe,42689,1731477133757 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T05:52:14,337 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=5, maxPoolSize=5 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/bc386b7bf2fe:0, corePoolSize=10, maxPoolSize=10 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:14,338 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,340 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:14,340 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T05:52:14,340 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731477164340 2024-11-13T05:52:14,340 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T05:52:14,340 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,340 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T05:52:14,341 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T05:52:14,341 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T05:52:14,341 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T05:52:14,341 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T05:52:14,341 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T05:52:14,341 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,342 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T05:52:14,342 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T05:52:14,342 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T05:52:14,342 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T05:52:14,342 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T05:52:14,343 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477134343,5,FailOnTimeoutGroup] 2024-11-13T05:52:14,345 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477134343,5,FailOnTimeoutGroup] 2024-11-13T05:52:14,345 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,345 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T05:52:14,345 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,345 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,356 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T05:52:14,357 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:14,362 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:14,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T05:52:14,366 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:52:14,366 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:14,366 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T05:52:14,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:52:14,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:14,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T05:52:14,372 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:52:14,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:14,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T05:52:14,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:52:14,376 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(746): ClusterId : 16daaa54-bfac-4105-9f17-c066e86323d8 2024-11-13T05:52:14,376 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(746): ClusterId : 16daaa54-bfac-4105-9f17-c066e86323d8 2024-11-13T05:52:14,376 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:52:14,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:14,376 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:52:14,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:14,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:52:14,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:14,377 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:14,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:52:14,378 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:52:14,378 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:14,379 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:52:14,385 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:14,385 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74807229, jitterRate=0.11471457779407501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:14,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731477134362Initializing all the Stores at 1731477134362Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477134362Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477134363 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477134363Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477134363Cleaning up temporary data from old regions at 1731477134378 (+15 ms)Region opened successfully at 1731477134386 (+8 ms) 2024-11-13T05:52:14,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:52:14,387 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:52:14,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:52:14,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:52:14,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:52:14,388 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:14,388 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477134387Disabling compacts and flushes for region at 1731477134387Disabling writes for close at 1731477134387Writing region close event to WAL at 1731477134388 (+1 ms)Closed at 1731477134388 2024-11-13T05:52:14,389 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:14,389 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T05:52:14,389 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:52:14,390 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:52:14,391 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:52:14,391 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:14,391 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:52:14,391 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:52:14,392 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T05:52:14,407 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:52:14,407 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:52:14,408 DEBUG [RS:0;bc386b7bf2fe:41307 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e58b03d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:14,408 DEBUG [RS:1;bc386b7bf2fe:35173 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3676468b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:14,421 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;bc386b7bf2fe:41307 2024-11-13T05:52:14,421 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:52:14,421 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:52:14,421 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:52:14,422 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,42689,1731477133757 with port=41307, startcode=1731477133907 2024-11-13T05:52:14,423 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;bc386b7bf2fe:35173 2024-11-13T05:52:14,423 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:52:14,423 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:52:14,423 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:52:14,423 DEBUG [RS:0;bc386b7bf2fe:41307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:52:14,424 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,42689,1731477133757 with port=35173, startcode=1731477133947 2024-11-13T05:52:14,424 DEBUG [RS:1;bc386b7bf2fe:35173 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:52:14,425 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33015, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:52:14,426 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40343, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:52:14,426 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,426 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,428 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,428 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:14,428 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T05:52:14,428 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:52:14,430 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:14,430 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T05:52:14,430 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:52:14,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:14,471 DEBUG [RS:0;bc386b7bf2fe:41307 {}] zookeeper.ZKUtil(111): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,471 WARN [RS:0;bc386b7bf2fe:41307 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:14,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,35173,1731477133947] 2024-11-13T05:52:14,471 INFO [RS:0;bc386b7bf2fe:41307 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:14,472 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,41307,1731477133907] 2024-11-13T05:52:14,472 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,472 DEBUG [RS:1;bc386b7bf2fe:35173 {}] zookeeper.ZKUtil(111): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,472 WARN [RS:1;bc386b7bf2fe:35173 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:14,472 INFO [RS:1;bc386b7bf2fe:35173 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:14,472 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,479 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:52:14,479 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:52:14,482 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:52:14,482 INFO [RS:0;bc386b7bf2fe:41307 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:52:14,482 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,483 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:52:14,483 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:52:14,484 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,484 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,484 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,484 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:52:14,484 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,484 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 INFO [RS:1;bc386b7bf2fe:35173 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,485 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,486 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,486 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:14,486 DEBUG [RS:0;bc386b7bf2fe:41307 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:14,486 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:52:14,487 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:52:14,487 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,487 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,41307,1731477133907-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,487 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:14,488 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:14,489 DEBUG [RS:1;bc386b7bf2fe:35173 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,490 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,35173,1731477133947-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:14,501 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:52:14,501 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,41307,1731477133907-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,502 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,502 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.Replication(171): bc386b7bf2fe,41307,1731477133907 started 2024-11-13T05:52:14,504 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:52:14,504 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,35173,1731477133947-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,504 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,504 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.Replication(171): bc386b7bf2fe,35173,1731477133947 started 2024-11-13T05:52:14,515 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,515 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,41307,1731477133907, RpcServer on bc386b7bf2fe/172.17.0.2:41307, sessionid=0x1013409ec730001 2024-11-13T05:52:14,515 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:52:14,515 DEBUG [RS:0;bc386b7bf2fe:41307 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,515 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,41307,1731477133907' 2024-11-13T05:52:14,515 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:52:14,516 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:52:14,516 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:52:14,516 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:52:14,517 DEBUG [RS:0;bc386b7bf2fe:41307 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,517 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,41307,1731477133907' 2024-11-13T05:52:14,517 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:52:14,517 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:52:14,518 DEBUG [RS:0;bc386b7bf2fe:41307 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:52:14,518 INFO [RS:0;bc386b7bf2fe:41307 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:52:14,518 INFO [RS:0;bc386b7bf2fe:41307 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:52:14,518 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:14,518 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,35173,1731477133947, RpcServer on bc386b7bf2fe/172.17.0.2:35173, sessionid=0x1013409ec730002 2024-11-13T05:52:14,519 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:52:14,519 DEBUG [RS:1;bc386b7bf2fe:35173 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,519 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,35173,1731477133947' 2024-11-13T05:52:14,519 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:52:14,519 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,35173,1731477133947' 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:52:14,520 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:52:14,521 DEBUG [RS:1;bc386b7bf2fe:35173 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:52:14,521 INFO [RS:1;bc386b7bf2fe:35173 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:52:14,521 INFO [RS:1;bc386b7bf2fe:35173 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:52:14,542 WARN [bc386b7bf2fe:42689 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T05:52:14,620 INFO [RS:0;bc386b7bf2fe:41307 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:14,623 INFO [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C41307%2C1731477133907, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:14,623 INFO [RS:1;bc386b7bf2fe:35173 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:14,624 INFO [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C35173%2C1731477133947, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,35173,1731477133947, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:14,642 INFO [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,35173,1731477133947/bc386b7bf2fe%2C35173%2C1731477133947.1731477134627 2024-11-13T05:52:14,643 DEBUG [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:14,647 INFO [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907/bc386b7bf2fe%2C41307%2C1731477133907.1731477134627 2024-11-13T05:52:14,647 DEBUG [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:14,792 DEBUG [bc386b7bf2fe:42689 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T05:52:14,793 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:52:14,794 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:52:14,795 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:52:14,795 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:52:14,795 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:52:14,795 INFO [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:52:14,795 INFO [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:52:14,795 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:52:14,796 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:14,798 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,41307,1731477133907, state=OPENING 2024-11-13T05:52:14,871 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T05:52:14,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:14,882 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:14,883 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,41307,1731477133907}] 2024-11-13T05:52:14,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:14,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:14,883 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:15,038 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:52:15,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39033, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:52:15,046 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:52:15,046 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:15,047 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:52:15,048 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C41307%2C1731477133907.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:15,068 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907/bc386b7bf2fe%2C41307%2C1731477133907.meta.1731477135049.meta 2024-11-13T05:52:15,068 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:15,068 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:15,069 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:52:15,069 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:52:15,069 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:52:15,069 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:52:15,069 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:15,070 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:52:15,070 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:52:15,071 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T05:52:15,071 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:52:15,071 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:15,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T05:52:15,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:52:15,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,072 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:15,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T05:52:15,073 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:52:15,073 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:15,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T05:52:15,074 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:52:15,074 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:15,074 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:52:15,075 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:15,075 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:15,076 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:52:15,076 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:52:15,076 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:15,077 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:52:15,078 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70587032, jitterRate=0.051828742027282715}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:15,078 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:52:15,079 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477135070Writing region info on filesystem at 1731477135070Initializing all the Stores at 1731477135070Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477135070Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477135071 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477135071Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477135071Cleaning up temporary data from old regions at 1731477135076 (+5 ms)Running coprocessor post-open hooks at 1731477135078 (+2 ms)Region opened successfully at 1731477135078 2024-11-13T05:52:15,079 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731477135037 2024-11-13T05:52:15,082 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:52:15,082 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:52:15,083 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:15,084 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,41307,1731477133907, state=OPEN 2024-11-13T05:52:15,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:15,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:15,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:15,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:15,105 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:15,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:15,105 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:15,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T05:52:15,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,41307,1731477133907 in 223 msec 2024-11-13T05:52:15,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T05:52:15,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 720 msec 2024-11-13T05:52:15,113 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T05:52:15,114 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T05:52:15,117 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:15,117 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1] 2024-11-13T05:52:15,117 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:15,119 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54387, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:15,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 789 msec 2024-11-13T05:52:15,124 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731477135124, completionTime=-1 2024-11-13T05:52:15,125 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-13T05:52:15,125 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T05:52:15,127 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731477195128 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731477255128 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,128 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,129 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-bc386b7bf2fe:42689, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,129 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,129 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,132 DEBUG [master/bc386b7bf2fe:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.119sec 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T05:52:15,134 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:15,135 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T05:52:15,136 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T05:52:15,136 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T05:52:15,136 INFO [master/bc386b7bf2fe:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,42689,1731477133757-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:15,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:15,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request bc386b7bf2fe,42689,-1 for getting cluster id 2024-11-13T05:52:15,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T05:52:15,179 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '16daaa54-bfac-4105-9f17-c066e86323d8' 2024-11-13T05:52:15,180 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T05:52:15,180 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "16daaa54-bfac-4105-9f17-c066e86323d8" 2024-11-13T05:52:15,180 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197b6262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:15,181 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [bc386b7bf2fe,42689,-1] 2024-11-13T05:52:15,181 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T05:52:15,181 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:15,183 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53072, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T05:52:15,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46dfee43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T05:52:15,186 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:15,188 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1] 2024-11-13T05:52:15,188 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:15,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:15,194 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:15,195 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T05:52:15,196 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:15,196 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a906cfb 2024-11-13T05:52:15,196 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T05:52:15,198 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T05:52:15,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731477135194', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T05:52:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 2024-11-13T05:52:15,201 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T05:52:15,201 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731477135194" procId is: 4 2024-11-13T05:52:15,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T05:52:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:15,212 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4de3452f76c01609525255aa4071c563, NAME => 'testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731477135194', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:15,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:15,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1722): Closing 4de3452f76c01609525255aa4071c563, disabling compactions & flushes 2024-11-13T05:52:15,218 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,219 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. after waiting 0 ms 2024-11-13T05:52:15,219 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,219 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,219 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731477135194-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4de3452f76c01609525255aa4071c563: Waiting for close lock at 1731477135218Disabling compacts and flushes for region at 1731477135218Disabling writes for close at 1731477135219 (+1 ms)Writing region close event to WAL at 1731477135219Closed at 1731477135219 2024-11-13T05:52:15,220 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T05:52:15,220 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731477135220"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731477135220"}]},"ts":"1731477135220"} 2024-11-13T05:52:15,223 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T05:52:15,224 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T05:52:15,224 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731477135194","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477135224"}]},"ts":"1731477135224"} 2024-11-13T05:52:15,226 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731477135194, state=ENABLING in hbase:meta 2024-11-13T05:52:15,226 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:52:15,227 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:52:15,227 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:52:15,227 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:52:15,227 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:52:15,227 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:52:15,227 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:52:15,227 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:52:15,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN}] 2024-11-13T05:52:15,229 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN 2024-11-13T05:52:15,230 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN; state=OFFLINE, location=bc386b7bf2fe,41307,1731477133907; forceNewPlan=false, retain=false 2024-11-13T05:52:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:15,380 INFO [bc386b7bf2fe:42689 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T05:52:15,380 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4de3452f76c01609525255aa4071c563, regionState=OPENING, regionLocation=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:15,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN because future has completed 2024-11-13T05:52:15,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,41307,1731477133907}] 2024-11-13T05:52:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:15,542 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,542 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4de3452f76c01609525255aa4071c563, NAME => 'testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:15,543 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731477135194 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,543 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:15,543 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,544 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,545 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,551 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4de3452f76c01609525255aa4071c563 columnFamilyName family 2024-11-13T05:52:15,551 DEBUG [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:15,551 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] regionserver.HStore(327): Store=4de3452f76c01609525255aa4071c563/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:15,551 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,552 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,552 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,552 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,552 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,553 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,558 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T05:52:15,558 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4de3452f76c01609525255aa4071c563; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66788473, jitterRate=-0.004774197936058044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:52:15,558 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:15,559 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4de3452f76c01609525255aa4071c563: Running coprocessor pre-open hook at 1731477135544Writing region info on filesystem at 1731477135544Initializing all the Stores at 1731477135544Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477135544Cleaning up temporary data from old regions at 1731477135552 (+8 ms)Running coprocessor post-open hooks at 1731477135559 (+7 ms)Region opened successfully at 1731477135559 2024-11-13T05:52:15,560 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563., pid=6, masterSystemTime=1731477135537 2024-11-13T05:52:15,562 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,562 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:15,563 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4de3452f76c01609525255aa4071c563, regionState=OPEN, openSeqNum=2, regionLocation=bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:15,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,41307,1731477133907 because future has completed 2024-11-13T05:52:15,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T05:52:15,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,41307,1731477133907 in 183 msec 2024-11-13T05:52:15,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T05:52:15,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN in 342 msec 2024-11-13T05:52:15,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T05:52:15,572 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731477135194","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731477135572"}]},"ts":"1731477135572"} 2024-11-13T05:52:15,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731477135194, state=ENABLED in hbase:meta 2024-11-13T05:52:15,575 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T05:52:15,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731477135194 in 376 msec 2024-11-13T05:52:15,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42689 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T05:52:15,835 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731477135194 completed 2024-11-13T05:52:15,835 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731477135194 get assigned. Timeout = 60000ms 2024-11-13T05:52:15,836 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:52:15,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731477135194 assigned to meta. Checking AM states. 2024-11-13T05:52:15,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T05:52:15,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731477135194 assigned. 2024-11-13T05:52:15,848 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731477135194', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563., hostname=bc386b7bf2fe,41307,1731477133907, seqNum=2] 2024-11-13T05:52:15,853 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4de3452f76c01609525255aa4071c563 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-13T05:52:15,861 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:15,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/.tmp/family/96155f9018ff433187af91e11e6ce281 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-13T05:52:15,876 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/.tmp/family/96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:15,879 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/.tmp/family/96155f9018ff433187af91e11e6ce281 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:15,881 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, delay=992 ms 2024-11-13T05:52:15,882 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, entries=2, sequenceid=6, filesize=4.9 K 2024-11-13T05:52:15,883 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4de3452f76c01609525255aa4071c563 in 31ms, sequenceid=6, compaction requested=false 2024-11-13T05:52:15,883 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731477135194' 2024-11-13T05:52:15,884 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4de3452f76c01609525255aa4071c563: 2024-11-13T05:52:15,921 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T05:52:16,875 DEBUG [hfile-prefetch-1731477133101 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 96155f9018ff433187af91e11e6ce281_0. 2024-11-13T05:52:16,876 DEBUG [hfile-prefetch-1731477133101 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 96155f9018ff433187af91e11e6ce281_117. 2024-11-13T05:52:16,877 DEBUG [hfile-prefetch-1731477133101 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, with total blocks 2, and data blocks 1 2024-11-13T05:52:16,877 DEBUG [hfile-prefetch-1731477133101 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-13T05:52:16,877 DEBUG [hfile-prefetch-1731477133101 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-13T05:52:16,877 DEBUG [hfile-prefetch-1731477133101 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281. Will sleep 100ms and try the verification again. 2024-11-13T05:52:16,884 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;bc386b7bf2fe:41307,5,FailOnTimeoutGroup] 2024-11-13T05:52:16,885 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,41307,1731477133907' ***** 2024-11-13T05:52:16,885 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-13T05:52:16,885 INFO [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T05:52:16,889 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:52:16,890 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:16,925 INFO [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence 2024-11-13T05:52:16,925 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T05:52:16,925 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T05:52:16,925 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T05:52:16,925 INFO [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T05:52:16,926 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T05:52:16,936 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1488): Thread RS:0;bc386b7bf2fe:41307 started persisting bucket cache to file 2024-11-13T05:52:16,936 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:16,936 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:16,945 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-13T05:52:16,946 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731477136936 2024-11-13T05:52:16,946 DEBUG [RS:0;bc386b7bf2fe:41307 {}] bucket.BucketCache(1510): Thread RS:0;bc386b7bf2fe:41307 finished persisting bucket cache to file, renaming 2024-11-13T05:52:16,946 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:52:16,946 INFO [RS:0;bc386b7bf2fe:41307 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:52:16,946 INFO [RS:0;bc386b7bf2fe:41307 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:52:16,947 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(3091): Received CLOSE for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:16,947 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:16,947 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:16,947 INFO [RS:0;bc386b7bf2fe:41307 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;bc386b7bf2fe:41307. 2024-11-13T05:52:16,947 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:16,947 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4de3452f76c01609525255aa4071c563, disabling compactions & flushes 2024-11-13T05:52:16,947 DEBUG [RS:0;bc386b7bf2fe:41307 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:16,947 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:16,947 DEBUG [RS:0;bc386b7bf2fe:41307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:16,948 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:16,948 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. after waiting 0 ms 2024-11-13T05:52:16,948 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:52:16,948 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:16,948 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:52:16,948 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:52:16,948 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:52:16,948 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:52:16,948 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4de3452f76c01609525255aa4071c563=testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.} 2024-11-13T05:52:16,949 DEBUG [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:16,949 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:16,949 DEBUG [StoreFileCloser-4de3452f76c01609525255aa4071c563-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:16,949 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:52:16,949 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:52:16,949 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:52:16,949 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:52:16,949 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:52:16,949 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-13T05:52:16,952 WARN [hfile-prefetch-1731477133101 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-13T05:52:16,953 DEBUG [hfile-prefetch-1731477133101 {}] hfile.PrefetchExecutor(121): Prefetch completed for 96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:16,954 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-13T05:52:16,955 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:16,955 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4de3452f76c01609525255aa4071c563: Waiting for close lock at 1731477136947Running coprocessor pre-close hooks at 1731477136947Disabling compacts and flushes for region at 1731477136947Disabling writes for close at 1731477136948 (+1 ms)Writing region close event to WAL at 1731477136949 (+1 ms)Running coprocessor post-close hooks at 1731477136955 (+6 ms)Closed at 1731477136955 2024-11-13T05:52:16,955 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:16,960 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:16,968 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-13T05:52:16,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-13T05:52:16,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:16,968 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:16,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:16,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477136968 2024-11-13T05:52:16,971 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-13T05:52:16,980 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/57583e8cf6544ec9a9a52e655796960b is 209, key is testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563./info:regioninfo/1731477135562/Put/seqid=0 2024-11-13T05:52:16,980 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:16,989 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:17,002 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/e4fe18c361724debb8f5d40bd03fbf33 is 43, key is default/ns:d/1731477135120/Put/seqid=0 2024-11-13T05:52:17,002 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:17,011 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:17,023 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/c8fc12e217114e8d90806579adaf16fa is 80, key is testBlockEvictionOnGracefulStop1731477135194/table:state/1731477135572/Put/seqid=0 2024-11-13T05:52:17,024 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:17,026 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/57583e8cf6544ec9a9a52e655796960b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:17,027 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b, delay=953 ms 2024-11-13T05:52:17,028 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b, entries=10, sequenceid=11, filesize=7.0 K 2024-11-13T05:52:17,029 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/ns/e4fe18c361724debb8f5d40bd03fbf33 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:17,030 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33, delay=971 ms 2024-11-13T05:52:17,031 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33, entries=2, sequenceid=11, filesize=5.0 K 2024-11-13T05:52:17,031 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/table/c8fc12e217114e8d90806579adaf16fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:17,032 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa, delay=989 ms 2024-11-13T05:52:17,033 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa, entries=2, sequenceid=11, filesize=5.3 K 2024-11-13T05:52:17,034 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false 2024-11-13T05:52:17,034 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:17,035 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:17,036 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:17,040 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-13T05:52:17,040 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:52:17,040 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:17,040 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477136948Running coprocessor pre-close hooks at 1731477136949 (+1 ms)Disabling compacts and flushes for region at 1731477136949Disabling writes for close at 1731477136949Obtaining lock to block concurrent updates at 1731477136949Preparing flush snapshotting stores in 1588230740 at 1731477136949Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731477136950 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731477136950Flushing 1588230740/info: creating writer at 1731477136950Flushing 1588230740/info: appending metadata at 1731477136979 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731477136979Flushing 1588230740/ns: creating writer at 1731477136983 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731477137001 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731477137001Flushing 1588230740/table: creating writer at 1731477137005 (+4 ms)Flushing 1588230740/table: appending metadata at 1731477137023 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731477137023Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@671c31f5: reopening flushed file at 1731477137026 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d47a9c0: reopening flushed file at 1731477137029 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4dda6320: reopening flushed file at 1731477137031 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 85ms, sequenceid=11, compaction requested=false at 1731477137034 (+3 ms)Writing region close event to WAL at 1731477137036 (+2 ms)Running coprocessor post-close hooks at 1731477137040 (+4 ms)Closed at 1731477137040 2024-11-13T05:52:17,040 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:17,149 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,41307,1731477133907; all regions closed. 2024-11-13T05:52:17,152 DEBUG [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:17,152 INFO [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C41307%2C1731477133907.meta:.meta(num 1731477135049) 2024-11-13T05:52:17,154 DEBUG [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:17,154 INFO [RS:0;bc386b7bf2fe:41307 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C41307%2C1731477133907:(num 1731477134627) 2024-11-13T05:52:17,154 DEBUG [RS:0;bc386b7bf2fe:41307 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:17,154 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:17,154 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:17,154 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:17,154 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:17,154 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:17,155 INFO [RS:0;bc386b7bf2fe:41307 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41307 2024-11-13T05:52:17,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,41307,1731477133907 2024-11-13T05:52:17,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:17,181 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:17,190 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,41307,1731477133907] 2024-11-13T05:52:17,198 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,41307,1731477133907 already deleted, retry=false 2024-11-13T05:52:17,198 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of bc386b7bf2fe,41307,1731477133907 on bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:17,204 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure bc386b7bf2fe,41307,1731477133907, splitWal=true, meta=true 2024-11-13T05:52:17,207 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure bc386b7bf2fe,41307,1731477133907, splitWal=true, meta=true 2024-11-13T05:52:17,207 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for bc386b7bf2fe,41307,1731477133907 (carryingMeta=true) bc386b7bf2fe,41307,1731477133907/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@434191c7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T05:52:17,208 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,41307,1731477133907, splitWal=true, meta=true, isMeta: true 2024-11-13T05:52:17,209 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907-splitting 2024-11-13T05:52:17,209 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907-splitting dir is empty, no logs to split. 2024-11-13T05:52:17,209 INFO [PEWorker-2 {}] master.SplitWALManager(105): bc386b7bf2fe,41307,1731477133907 WAL count=0, meta=true 2024-11-13T05:52:17,211 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907-splitting dir is empty, no logs to split. 2024-11-13T05:52:17,211 INFO [PEWorker-2 {}] master.SplitWALManager(105): bc386b7bf2fe,41307,1731477133907 WAL count=0, meta=true 2024-11-13T05:52:17,212 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,41307,1731477133907 WAL splitting is done? wals=0, meta=true 2024-11-13T05:52:17,214 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T05:52:17,215 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:17,216 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:52:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:17,290 INFO [RS:0;bc386b7bf2fe:41307 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:17,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41307-0x1013409ec730001, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:17,290 INFO [RS:0;bc386b7bf2fe:41307 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,41307,1731477133907; zookeeper connection closed. 2024-11-13T05:52:17,291 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7a9523d0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7a9523d0 2024-11-13T05:52:17,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T05:52:17,311 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T05:52:17,366 DEBUG [bc386b7bf2fe:42689 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T05:52:17,367 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:17,369 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,35173,1731477133947, state=OPENING 2024-11-13T05:52:17,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:17,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:17,397 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T05:52:17,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:17,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:17,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,35173,1731477133947}] 2024-11-13T05:52:17,408 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/bc386b7bf2fe:0 server-side Connection retries=45 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T05:52:17,409 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T05:52:17,410 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38353 2024-11-13T05:52:17,411 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38353 connecting to ZooKeeper ensemble=127.0.0.1:55047 2024-11-13T05:52:17,412 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:17,412 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:17,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:383530x0, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T05:52:17,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38353-0x1013409ec730003 connected 2024-11-13T05:52:17,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-13T05:52:17,423 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-13T05:52:17,424 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T05:52:17,425 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache 2024-11-13T05:52:17,425 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-13T05:52:17,425 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-13T05:52:17,425 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-13T05:52:17,426 INFO [Thread-208 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-13T05:52:17,426 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-13T05:52:17,426 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T05:52:17,427 INFO [Thread-208 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-13T05:52:17,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T05:52:17,427 INFO [Thread-208 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-13T05:52:17,429 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T05:52:17,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-13T05:52:17,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38353 2024-11-13T05:52:17,434 DEBUG [Thread-208 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-13T05:52:17,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38353 2024-11-13T05:52:17,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-13T05:52:17,434 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38353 2024-11-13T05:52:17,434 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-13T05:52:17,434 INFO [Thread-208 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-13T05:52:17,434 INFO [Thread-208 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence is complete. 2024-11-13T05:52:17,436 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(746): ClusterId : 16daaa54-bfac-4105-9f17-c066e86323d8 2024-11-13T05:52:17,436 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T05:52:17,449 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T05:52:17,449 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T05:52:17,457 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T05:52:17,457 DEBUG [RS:2;bc386b7bf2fe:38353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5eb5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=bc386b7bf2fe/172.17.0.2:0 2024-11-13T05:52:17,469 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;bc386b7bf2fe:38353 2024-11-13T05:52:17,469 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T05:52:17,469 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T05:52:17,469 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T05:52:17,470 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(2659): reportForDuty to master=bc386b7bf2fe,42689,1731477133757 with port=38353, startcode=1731477137408 2024-11-13T05:52:17,470 DEBUG [RS:2;bc386b7bf2fe:38353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T05:52:17,472 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T05:52:17,473 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(363): Checking decommissioned status of RegionServer bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,473 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42689 {}] master.ServerManager(517): Registering regionserver=bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,474 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2 2024-11-13T05:52:17,474 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-13T05:52:17,475 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T05:52:17,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:17,481 DEBUG [RS:2;bc386b7bf2fe:38353 {}] zookeeper.ZKUtil(111): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,482 WARN [RS:2;bc386b7bf2fe:38353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T05:52:17,482 INFO [RS:2;bc386b7bf2fe:38353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:17,482 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [bc386b7bf2fe,38353,1731477137408] 2024-11-13T05:52:17,482 DEBUG [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,486 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T05:52:17,489 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T05:52:17,489 INFO [RS:2;bc386b7bf2fe:38353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T05:52:17,489 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,489 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T05:52:17,490 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T05:52:17,490 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=2, maxPoolSize=2 2024-11-13T05:52:17,490 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/bc386b7bf2fe:0, corePoolSize=1, maxPoolSize=1 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:17,491 DEBUG [RS:2;bc386b7bf2fe:38353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/bc386b7bf2fe:0, corePoolSize=3, maxPoolSize=3 2024-11-13T05:52:17,491 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,491 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,491 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,492 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,492 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,492 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38353,1731477137408-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T05:52:17,507 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T05:52:17,508 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=bc386b7bf2fe,38353,1731477137408-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,508 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,508 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.Replication(171): bc386b7bf2fe,38353,1731477137408 started 2024-11-13T05:52:17,521 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T05:52:17,521 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1482): Serving as bc386b7bf2fe,38353,1731477137408, RpcServer on bc386b7bf2fe/172.17.0.2:38353, sessionid=0x1013409ec730003 2024-11-13T05:52:17,521 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T05:52:17,521 DEBUG [RS:2;bc386b7bf2fe:38353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,521 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38353,1731477137408' 2024-11-13T05:52:17,521 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T05:52:17,522 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T05:52:17,522 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T05:52:17,522 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T05:52:17,522 DEBUG [RS:2;bc386b7bf2fe:38353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:17,522 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'bc386b7bf2fe,38353,1731477137408' 2024-11-13T05:52:17,523 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T05:52:17,523 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T05:52:17,523 DEBUG [RS:2;bc386b7bf2fe:38353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T05:52:17,523 INFO [RS:2;bc386b7bf2fe:38353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T05:52:17,524 INFO [RS:2;bc386b7bf2fe:38353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T05:52:17,551 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T05:52:17,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53245, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T05:52:17,554 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T05:52:17,554 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T05:52:17,555 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T05:52:17,555 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C35173%2C1731477133947.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,35173,1731477133947, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:17,572 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,35173,1731477133947/bc386b7bf2fe%2C35173%2C1731477133947.meta.1731477137556.meta 2024-11-13T05:52:17,573 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:17,573 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:17,573 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T05:52:17,573 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T05:52:17,574 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T05:52:17,574 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T05:52:17,574 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:17,574 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T05:52:17,574 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T05:52:17,575 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-13T05:52:17,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T05:52:17,576 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:17,578 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b, delay=996 ms 2024-11-13T05:52:17,579 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:17,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:17,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-13T05:52:17,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T05:52:17,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:17,582 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33, delay=948 ms 2024-11-13T05:52:17,583 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:17,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:17,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-13T05:52:17,584 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T05:52:17,584 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:17,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:17,585 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-13T05:52:17,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T05:52:17,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:17,587 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa, delay=974 ms 2024-11-13T05:52:17,588 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:17,588 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T05:52:17,588 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T05:52:17,589 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:17,589 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740 2024-11-13T05:52:17,590 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T05:52:17,590 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T05:52:17,590 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T05:52:17,591 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T05:52:17,591 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74574317, jitterRate=0.11124391853809357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T05:52:17,592 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T05:52:17,592 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731477137574Writing region info on filesystem at 1731477137574Initializing all the Stores at 1731477137575 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477137575Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477137575Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477137575Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731477137575Cleaning up temporary data from old regions at 1731477137590 (+15 ms)Running coprocessor post-open hooks at 1731477137592 (+2 ms)Region opened successfully at 1731477137592 2024-11-13T05:52:17,593 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731477137550 2024-11-13T05:52:17,597 DEBUG [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T05:52:17,597 INFO [RS_OPEN_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T05:52:17,597 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:17,598 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as bc386b7bf2fe,35173,1731477133947, state=OPEN 2024-11-13T05:52:17,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:17,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:17,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T05:52:17,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:17,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:17,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:17,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T05:52:17,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-13T05:52:17,615 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=bc386b7bf2fe,35173,1731477133947 in 216 msec 2024-11-13T05:52:17,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T05:52:17,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 402 msec 2024-11-13T05:52:17,617 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): bc386b7bf2fe,41307,1731477133907 had 2 regions 2024-11-13T05:52:17,618 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure bc386b7bf2fe,41307,1731477133907, splitWal=true, meta=true, isMeta: false 2024-11-13T05:52:17,618 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907-splitting dir is empty, no logs to split. 2024-11-13T05:52:17,618 INFO [PEWorker-3 {}] master.SplitWALManager(105): bc386b7bf2fe,41307,1731477133907 WAL count=0, meta=false 2024-11-13T05:52:17,619 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,41307,1731477133907-splitting dir is empty, no logs to split. 2024-11-13T05:52:17,619 INFO [PEWorker-3 {}] master.SplitWALManager(105): bc386b7bf2fe,41307,1731477133907 WAL count=0, meta=false 2024-11-13T05:52:17,619 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if bc386b7bf2fe,41307,1731477133907 WAL splitting is done? wals=0, meta=false 2024-11-13T05:52:17,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN}] 2024-11-13T05:52:17,622 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN 2024-11-13T05:52:17,622 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T05:52:17,624 INFO [RS:2;bc386b7bf2fe:38353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T05:52:17,625 INFO [RS:2;bc386b7bf2fe:38353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=bc386b7bf2fe%2C38353%2C1731477137408, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,38353,1731477137408, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs, maxLogs=55 2024-11-13T05:52:17,643 INFO [RS:2;bc386b7bf2fe:38353 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/WALs/bc386b7bf2fe,38353,1731477137408/bc386b7bf2fe%2C38353%2C1731477137408.1731477137625 2024-11-13T05:52:17,643 DEBUG [RS:2;bc386b7bf2fe:38353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(204): Hosts are {bc386b7bf2fe=0} racks are {/default-rack=0} 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T05:52:17,773 INFO [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T05:52:17,773 INFO [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T05:52:17,773 DEBUG [bc386b7bf2fe:42689 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T05:52:17,774 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4de3452f76c01609525255aa4071c563, regionState=OPENING, regionLocation=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:17,775 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server bc386b7bf2fe:41307 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:41307 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T05:52:17,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1, error=java.net.ConnectException: Call to address=bc386b7bf2fe:41307 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:41307 2024-11-13T05:52:17,777 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T05:52:17,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,41307,1731477133907, seqNum=-1 from cache 2024-11-13T05:52:17,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address bc386b7bf2fe:41307 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: bc386b7bf2fe/172.17.0.2:41307 2024-11-13T05:52:17,883 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T05:52:17,883 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=bc386b7bf2fe,35173,1731477133947, seqNum=-1] 2024-11-13T05:52:17,884 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T05:52:17,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45321, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T05:52:17,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN because future has completed 2024-11-13T05:52:17,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,35173,1731477133947}] 2024-11-13T05:52:18,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T05:52:18,021 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:18,022 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:18,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T05:52:18,022 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T05:52:18,022 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=618753851, stopped=false 2024-11-13T05:52:18,023 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=bc386b7bf2fe,42689,1731477133757 2024-11-13T05:52:18,046 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,046 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 4de3452f76c01609525255aa4071c563, NAME => 'testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.', STARTKEY => '', ENDKEY => ''} 2024-11-13T05:52:18,048 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731477135194 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,048 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T05:52:18,048 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,048 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,051 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,051 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4de3452f76c01609525255aa4071c563 columnFamilyName family 2024-11-13T05:52:18,051 DEBUG [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T05:52:18,054 DEBUG [StoreFileOpener-4de3452f76c01609525255aa4071c563-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281, delay=995 ms 2024-11-13T05:52:18,055 DEBUG [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:18,055 INFO [StoreOpener-4de3452f76c01609525255aa4071c563-1 {}] regionserver.HStore(327): Store=4de3452f76c01609525255aa4071c563/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T05:52:18,056 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,056 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,057 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,057 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,057 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,058 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,059 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 4de3452f76c01609525255aa4071c563; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74035127, jitterRate=0.10320936143398285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T05:52:18,059 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,059 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 4de3452f76c01609525255aa4071c563: Running coprocessor pre-open hook at 1731477138050Writing region info on filesystem at 1731477138050Initializing all the Stores at 1731477138050Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731477138050Cleaning up temporary data from old regions at 1731477138057 (+7 ms)Running coprocessor post-open hooks at 1731477138059 (+2 ms)Region opened successfully at 1731477138059 2024-11-13T05:52:18,060 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563., pid=11, masterSystemTime=1731477138042 2024-11-13T05:52:18,063 DEBUG [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,063 INFO [RS_OPEN_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=4de3452f76c01609525255aa4071c563, regionState=OPEN, openSeqNum=10, regionLocation=bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:18,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,35173,1731477133947 because future has completed 2024-11-13T05:52:18,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T05:52:18,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 4de3452f76c01609525255aa4071c563, server=bc386b7bf2fe,35173,1731477133947 in 178 msec 2024-11-13T05:52:18,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-13T05:52:18,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731477135194, region=4de3452f76c01609525255aa4071c563, ASSIGN in 448 msec 2024-11-13T05:52:18,071 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server bc386b7bf2fe,41307,1731477133907 after splitting done 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:18,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:18,073 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:52:18,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure bc386b7bf2fe,41307,1731477133907, splitWal=true, meta=true in 872 msec 2024-11-13T05:52:18,073 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T05:52:18,073 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:18,073 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,073 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:18,073 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:18,073 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,35173,1731477133947' ***** 2024-11-13T05:52:18,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:52:18,074 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'bc386b7bf2fe,38353,1731477137408' ***** 2024-11-13T05:52:18,074 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T05:52:18,074 INFO [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T05:52:18,074 INFO [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1739): Disabling cache 2024-11-13T05:52:18,074 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T05:52:18,074 INFO [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T05:52:18,074 INFO [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/246c9ce6-d494-d6c8-0a41-6e852721a568/bucket.persistence 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-13T05:52:18,074 INFO [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T05:52:18,074 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-13T05:52:18,075 INFO [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-13T05:52:18,075 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T05:52:18,075 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-13T05:52:18,085 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1488): Thread RS:1;bc386b7bf2fe:35173 started persisting bucket cache to file 2024-11-13T05:52:18,085 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1488): Thread RS:2;bc386b7bf2fe:38353 started persisting bucket cache to file 2024-11-13T05:52:18,085 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:18,085 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-13T05:52:18,085 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:18,085 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-13T05:52:18,088 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:18,088 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-13T05:52:18,088 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477138085 2024-11-13T05:52:18,088 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731477138085 2024-11-13T05:52:18,088 DEBUG [RS:1;bc386b7bf2fe:35173 {}] bucket.BucketCache(1510): Thread RS:1;bc386b7bf2fe:35173 finished persisting bucket cache to file, renaming 2024-11-13T05:52:18,088 DEBUG [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1510): Thread RS:2;bc386b7bf2fe:38353 finished persisting bucket cache to file, renaming 2024-11-13T05:52:18,088 WARN [RS:2;bc386b7bf2fe:38353 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-13T05:52:18,088 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:52:18,088 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T05:52:18,088 INFO [RS:2;bc386b7bf2fe:38353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:52:18,088 INFO [RS:1;bc386b7bf2fe:35173 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T05:52:18,089 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:52:18,089 INFO [RS:2;bc386b7bf2fe:38353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T05:52:18,089 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T05:52:18,089 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(3091): Received CLOSE for 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,089 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:18,089 INFO [RS:2;bc386b7bf2fe:38353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;bc386b7bf2fe:38353. 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(959): stopping server bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:18,089 DEBUG [RS:2;bc386b7bf2fe:38353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:18,089 DEBUG [RS:2;bc386b7bf2fe:38353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;bc386b7bf2fe:35173. 2024-11-13T05:52:18,089 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:18,089 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,38353,1731477137408; all regions closed. 2024-11-13T05:52:18,089 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4de3452f76c01609525255aa4071c563, disabling compactions & flushes 2024-11-13T05:52:18,089 DEBUG [RS:1;bc386b7bf2fe:35173 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T05:52:18,089 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,089 DEBUG [RS:1;bc386b7bf2fe:35173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,089 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,089 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. after waiting 0 ms 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:52:18,089 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:52:18,089 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:52:18,090 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T05:52:18,091 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T05:52:18,091 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 4de3452f76c01609525255aa4071c563=testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563.} 2024-11-13T05:52:18,091 DEBUG [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4de3452f76c01609525255aa4071c563 2024-11-13T05:52:18,091 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-13T05:52:18,091 DEBUG [StoreFileCloser-4de3452f76c01609525255aa4071c563-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/family/96155f9018ff433187af91e11e6ce281 2024-11-13T05:52:18,091 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T05:52:18,091 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T05:52:18,091 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T05:52:18,091 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T05:52:18,091 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T05:52:18,091 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-13T05:52:18,093 DEBUG [RS:2;bc386b7bf2fe:38353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:18,093 INFO [RS:2;bc386b7bf2fe:38353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C38353%2C1731477137408:(num 1731477137625) 2024-11-13T05:52:18,093 DEBUG [RS:2;bc386b7bf2fe:38353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,093 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:18,093 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:18,093 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:18,093 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T05:52:18,094 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T05:52:18,094 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:18,094 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T05:52:18,094 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:18,094 INFO [RS:2;bc386b7bf2fe:38353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38353 2024-11-13T05:52:18,094 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:18,094 INFO [regionserver/bc386b7bf2fe:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:18,095 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/default/testBlockEvictionOnGracefulStop1731477135194/4de3452f76c01609525255aa4071c563/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-13T05:52:18,096 INFO [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,096 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4de3452f76c01609525255aa4071c563: Waiting for close lock at 1731477138089Running coprocessor pre-close hooks at 1731477138089Disabling compacts and flushes for region at 1731477138089Disabling writes for close at 1731477138089Writing region close event to WAL at 1731477138091 (+2 ms)Running coprocessor post-close hooks at 1731477138096 (+5 ms)Closed at 1731477138096 2024-11-13T05:52:18,096 DEBUG [RS_CLOSE_REGION-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563. 2024-11-13T05:52:18,100 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f9aafa56ab344efc8c71620457f4de1e 2024-11-13T05:52:18,112 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/f9aafa56ab344efc8c71620457f4de1e is 209, key is testBlockEvictionOnGracefulStop1731477135194,,1731477135199.4de3452f76c01609525255aa4071c563./info:regioninfo/1731477138063/Put/seqid=0 2024-11-13T05:52:18,113 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/f9aafa56ab344efc8c71620457f4de1e 2024-11-13T05:52:18,115 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/.tmp/info/f9aafa56ab344efc8c71620457f4de1e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/f9aafa56ab344efc8c71620457f4de1e 2024-11-13T05:52:18,116 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/f9aafa56ab344efc8c71620457f4de1e, delay=961 ms 2024-11-13T05:52:18,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,38353,1731477137408 2024-11-13T05:52:18,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:18,116 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:18,117 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/f9aafa56ab344efc8c71620457f4de1e, entries=8, sequenceid=19, filesize=6.6 K 2024-11-13T05:52:18,117 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=19, compaction requested=false 2024-11-13T05:52:18,118 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/57583e8cf6544ec9a9a52e655796960b 2024-11-13T05:52:18,118 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/info/f9aafa56ab344efc8c71620457f4de1e 2024-11-13T05:52:18,119 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/ns/e4fe18c361724debb8f5d40bd03fbf33 2024-11-13T05:52:18,119 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/table/c8fc12e217114e8d90806579adaf16fa 2024-11-13T05:52:18,123 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-13T05:52:18,123 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T05:52:18,123 INFO [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:18,123 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731477138091Running coprocessor pre-close hooks at 1731477138091Disabling compacts and flushes for region at 1731477138091Disabling writes for close at 1731477138091Obtaining lock to block concurrent updates at 1731477138091Preparing flush snapshotting stores in 1588230740 at 1731477138091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731477138092 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731477138092Flushing 1588230740/info: creating writer at 1731477138092Flushing 1588230740/info: appending metadata at 1731477138112 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731477138112Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d7c163c: reopening flushed file at 1731477138115 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 26ms, sequenceid=19, compaction requested=false at 1731477138117 (+2 ms)Writing region close event to WAL at 1731477138119 (+2 ms)Running coprocessor post-close hooks at 1731477138123 (+4 ms)Closed at 1731477138123 2024-11-13T05:52:18,123 DEBUG [RS_CLOSE_META-regionserver/bc386b7bf2fe:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T05:52:18,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,38353,1731477137408] 2024-11-13T05:52:18,139 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,38353,1731477137408 already deleted, retry=false 2024-11-13T05:52:18,139 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,38353,1731477137408 expired; onlineServers=1 2024-11-13T05:52:18,229 INFO [RS:2;bc386b7bf2fe:38353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:18,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,229 INFO [RS:2;bc386b7bf2fe:38353 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,38353,1731477137408; zookeeper connection closed. 2024-11-13T05:52:18,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38353-0x1013409ec730003, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,230 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79dac47c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79dac47c 2024-11-13T05:52:18,291 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(976): stopping server bc386b7bf2fe,35173,1731477133947; all regions closed. 2024-11-13T05:52:18,293 DEBUG [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:18,293 INFO [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C35173%2C1731477133947.meta:.meta(num 1731477137556) 2024-11-13T05:52:18,294 DEBUG [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/oldWALs 2024-11-13T05:52:18,294 INFO [RS:1;bc386b7bf2fe:35173 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL bc386b7bf2fe%2C35173%2C1731477133947:(num 1731477134627) 2024-11-13T05:52:18,295 DEBUG [RS:1;bc386b7bf2fe:35173 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T05:52:18,295 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T05:52:18,295 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:18,295 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.ChoreService(370): Chore service for: regionserver/bc386b7bf2fe:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:18,295 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:18,295 INFO [regionserver/bc386b7bf2fe:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:18,295 INFO [RS:1;bc386b7bf2fe:35173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35173 2024-11-13T05:52:18,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/bc386b7bf2fe,35173,1731477133947 2024-11-13T05:52:18,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T05:52:18,321 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:18,331 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [bc386b7bf2fe,35173,1731477133947] 2024-11-13T05:52:18,339 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/bc386b7bf2fe,35173,1731477133947 already deleted, retry=false 2024-11-13T05:52:18,339 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; bc386b7bf2fe,35173,1731477133947 expired; onlineServers=0 2024-11-13T05:52:18,339 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'bc386b7bf2fe,42689,1731477133757' ***** 2024-11-13T05:52:18,339 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T05:52:18,339 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T05:52:18,339 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T05:52:18,340 DEBUG [M:0;bc386b7bf2fe:42689 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T05:52:18,340 DEBUG [M:0;bc386b7bf2fe:42689 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T05:52:18,340 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477134343 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.large.0-1731477134343,5,FailOnTimeoutGroup] 2024-11-13T05:52:18,340 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T05:52:18,340 DEBUG [master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477134343 {}] cleaner.HFileCleaner(306): Exit Thread[master/bc386b7bf2fe:0:becomeActiveMaster-HFileCleaner.small.0-1731477134343,5,FailOnTimeoutGroup] 2024-11-13T05:52:18,340 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.ChoreService(370): Chore service for: master/bc386b7bf2fe:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T05:52:18,340 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T05:52:18,340 DEBUG [M:0;bc386b7bf2fe:42689 {}] master.HMaster(1795): Stopping service threads 2024-11-13T05:52:18,340 INFO [M:0;bc386b7bf2fe:42689 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T05:52:18,340 INFO [M:0;bc386b7bf2fe:42689 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T05:52:18,341 INFO [M:0;bc386b7bf2fe:42689 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T05:52:18,341 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T05:52:18,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T05:52:18,347 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T05:52:18,347 DEBUG [M:0;bc386b7bf2fe:42689 {}] zookeeper.ZKUtil(347): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T05:52:18,347 WARN [M:0;bc386b7bf2fe:42689 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T05:52:18,348 INFO [M:0;bc386b7bf2fe:42689 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/.lastflushedseqids 2024-11-13T05:52:18,354 INFO [M:0;bc386b7bf2fe:42689 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T05:52:18,354 INFO [M:0;bc386b7bf2fe:42689 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T05:52:18,354 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T05:52:18,354 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:18,354 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:18,354 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T05:52:18,354 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:18,354 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-13T05:52:18,363 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c51899b48d9446c7b4d3502757c9d565 2024-11-13T05:52:18,374 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51899b48d9446c7b4d3502757c9d565 is 82, key is hbase:meta,,1/info:regioninfo/1731477137597/Put/seqid=0 2024-11-13T05:52:18,375 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51899b48d9446c7b4d3502757c9d565 2024-11-13T05:52:18,382 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ab7f7731688d48328458fcc6108c4b8e 2024-11-13T05:52:18,395 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab7f7731688d48328458fcc6108c4b8e is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731477135576/Put/seqid=0 2024-11-13T05:52:18,396 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab7f7731688d48328458fcc6108c4b8e 2024-11-13T05:52:18,402 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f7ab5f431e874745a637f84770429094 2024-11-13T05:52:18,415 DEBUG [M:0;bc386b7bf2fe:42689 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7ab5f431e874745a637f84770429094 is 69, key is bc386b7bf2fe,35173,1731477133947/rs:state/1731477134428/Put/seqid=0 2024-11-13T05:52:18,416 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7ab5f431e874745a637f84770429094 2024-11-13T05:52:18,417 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f7ab5f431e874745a637f84770429094 2024-11-13T05:52:18,418 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c51899b48d9446c7b4d3502757c9d565 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c51899b48d9446c7b4d3502757c9d565 2024-11-13T05:52:18,420 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c51899b48d9446c7b4d3502757c9d565, entries=14, sequenceid=102, filesize=5.9 K 2024-11-13T05:52:18,420 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ab7f7731688d48328458fcc6108c4b8e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ab7f7731688d48328458fcc6108c4b8e 2024-11-13T05:52:18,421 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ab7f7731688d48328458fcc6108c4b8e, entries=11, sequenceid=102, filesize=6.5 K 2024-11-13T05:52:18,422 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/f7ab5f431e874745a637f84770429094 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f7ab5f431e874745a637f84770429094 2024-11-13T05:52:18,423 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f7ab5f431e874745a637f84770429094 2024-11-13T05:52:18,423 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d677a745-c8da-0aaa-1c82-ad56437f7d39/eeeaa7a6-fce3-fcbe-43d0-bc71fe66a4d2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/f7ab5f431e874745a637f84770429094, entries=3, sequenceid=102, filesize=5.3 K 2024-11-13T05:52:18,424 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=102, compaction requested=false 2024-11-13T05:52:18,426 INFO [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T05:52:18,426 DEBUG [M:0;bc386b7bf2fe:42689 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731477138354Disabling compacts and flushes for region at 1731477138354Disabling writes for close at 1731477138354Obtaining lock to block concurrent updates at 1731477138354Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731477138354Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731477138355 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731477138355Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731477138355Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731477138374 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731477138374Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731477138376 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731477138395 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731477138395Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731477138397 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731477138414 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731477138414Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@600ffb7a: reopening flushed file at 1731477138418 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25ecb9f0: reopening flushed file at 1731477138420 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@705972b: reopening flushed file at 1731477138422 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=102, compaction requested=false at 1731477138424 (+2 ms)Writing region close event to WAL at 1731477138426 (+2 ms)Closed at 1731477138426 2024-11-13T05:52:18,427 INFO [M:0;bc386b7bf2fe:42689 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T05:52:18,427 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T05:52:18,427 INFO [M:0;bc386b7bf2fe:42689 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42689 2024-11-13T05:52:18,427 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T05:52:18,431 INFO [RS:1;bc386b7bf2fe:35173 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,431 INFO [RS:1;bc386b7bf2fe:35173 {}] regionserver.HRegionServer(1031): Exiting; stopping=bc386b7bf2fe,35173,1731477133947; zookeeper connection closed. 2024-11-13T05:52:18,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35173-0x1013409ec730002, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,431 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@378e1e01 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@378e1e01 2024-11-13T05:52:18,432 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T05:52:18,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,556 INFO [M:0;bc386b7bf2fe:42689 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T05:52:18,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42689-0x1013409ec730000, quorum=127.0.0.1:55047, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T05:52:18,560 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T05:52:18,563 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-13T05:52:18,563 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T05:52:18,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 366), ProcessCount=11 (was 11), AvailableMemoryMB=7271 (was 7347)