2024-11-23 22:45:30,912 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 22:45:30,941 main DEBUG Took 0.012683 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 22:45:30,942 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 22:45:30,942 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 22:45:30,943 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 22:45:30,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,955 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 22:45:30,972 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,974 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,975 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,977 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,978 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,979 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,980 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,982 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,982 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,983 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,984 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,984 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,985 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,986 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,987 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,987 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 22:45:30,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,988 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 22:45:30,990 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 22:45:30,991 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 22:45:30,993 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 22:45:30,994 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 22:45:30,996 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 22:45:30,997 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 22:45:31,006 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 22:45:31,009 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 22:45:31,011 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 22:45:31,011 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 22:45:31,012 main DEBUG createAppenders(={Console}) 2024-11-23 22:45:31,013 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-23 22:45:31,013 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 22:45:31,014 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-23 22:45:31,014 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 22:45:31,015 main DEBUG OutputStream closed 2024-11-23 22:45:31,015 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 22:45:31,015 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 22:45:31,016 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-23 22:45:31,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 22:45:31,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 22:45:31,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 22:45:31,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 22:45:31,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 22:45:31,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 22:45:31,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 22:45:31,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 22:45:31,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 22:45:31,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 22:45:31,107 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 22:45:31,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 22:45:31,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 22:45:31,108 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 22:45:31,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 22:45:31,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 22:45:31,109 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 22:45:31,110 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 22:45:31,113 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 22:45:31,113 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-23 22:45:31,113 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 22:45:31,114 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-23T22:45:31,130 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-23 22:45:31,133 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 22:45:31,134 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T22:45:31,378 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35 2024-11-23T22:45:31,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=453, ProcessCount=11, AvailableMemoryMB=7633 2024-11-23T22:45:31,498 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T22:45:31,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:31,593 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/cluster_3ef7b197-e974-7287-14e4-54bfe72c948d, deleteOnExit=true 2024-11-23T22:45:31,686 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/cluster_3ef7b197-e974-7287-14e4-54bfe72c948d/zookeeper_0, clientPort=57408, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/cluster_3ef7b197-e974-7287-14e4-54bfe72c948d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/cluster_3ef7b197-e974-7287-14e4-54bfe72c948d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T22:45:31,703 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57408 2024-11-23T22:45:31,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:31,804 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 with version=8 2024-11-23T22:45:31,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase-staging 2024-11-23T22:45:31,907 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T22:45:32,144 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:32,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:32,156 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:32,166 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:32,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:32,167 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:32,313 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T22:45:32,393 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T22:45:32,436 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T22:45:32,449 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T22:45:32,452 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:32,490 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 15254 (auto-detected) 2024-11-23T22:45:32,491 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T22:45:32,513 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42801 2024-11-23T22:45:32,535 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42801 connecting to ZooKeeper ensemble=127.0.0.1:57408 2024-11-23T22:45:32,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428010x0, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:32,651 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42801-0x10169dfa1350000 connected 2024-11-23T22:45:32,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:32,921 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:32,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:32,933 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9, hbase.cluster.distributed=false 2024-11-23T22:45:32,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:32,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-23T22:45:32,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42801 2024-11-23T22:45:32,970 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42801 2024-11-23T22:45:32,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-23T22:45:32,971 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42801 2024-11-23T22:45:33,048 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:33,049 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,050 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:33,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,050 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:33,053 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:45:33,056 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:33,057 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33605 2024-11-23T22:45:33,059 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33605 connecting to ZooKeeper ensemble=127.0.0.1:57408 2024-11-23T22:45:33,060 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:33,061 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:33,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:336050x0, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:33,084 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33605-0x10169dfa1350001 connected 2024-11-23T22:45:33,085 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:33,092 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:45:33,107 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.cache 2024-11-23T22:45:33,113 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T22:45:33,115 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T22:45:33,117 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T22:45:33,119 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T22:45:33,119 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T22:45:33,124 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T22:45:33,130 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:45:33,131 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T22:45:33,131 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence is complete. 2024-11-23T22:45:33,133 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:45:33,139 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:33,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33605 2024-11-23T22:45:33,141 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33605 2024-11-23T22:45:33,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33605 2024-11-23T22:45:33,147 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33605 2024-11-23T22:45:33,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33605 2024-11-23T22:45:33,168 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:33,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,169 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:33,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:33,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:33,170 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:45:33,170 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:33,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44275 2024-11-23T22:45:33,174 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44275 connecting to ZooKeeper ensemble=127.0.0.1:57408 2024-11-23T22:45:33,175 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:33,177 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:33,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442750x0, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:33,201 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442750x0, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:33,202 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:45:33,202 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44275-0x10169dfa1350002 connected 2024-11-23T22:45:33,203 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.cache 2024-11-23T22:45:33,204 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T22:45:33,204 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T22:45:33,204 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T22:45:33,205 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T22:45:33,205 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T22:45:33,205 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T22:45:33,206 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T22:45:33,206 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence is complete. 2024-11-23T22:45:33,208 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:45:33,210 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:45:33,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:33,216 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44275 2024-11-23T22:45:33,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44275 2024-11-23T22:45:33,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44275 2024-11-23T22:45:33,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44275 2024-11-23T22:45:33,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44275 2024-11-23T22:45:33,241 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:42801 2024-11-23T22:45:33,242 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:33,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,257 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:33,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:33,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:33,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,297 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T22:45:33,298 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,42801,1732401931984 from backup master directory 2024-11-23T22:45:33,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:33,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:33,316 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:33,316 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:33,319 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T22:45:33,324 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T22:45:33,335 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase.id] with ID: c5ac0603-26c7-4672-8afc-f831b0d0ab7b 2024-11-23T22:45:33,335 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.tmp/hbase.id 2024-11-23T22:45:33,340 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase.id] 2024-11-23T22:45:33,387 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:33,394 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T22:45:33,417 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-11-23T22:45:33,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,432 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:33,470 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:45:33,475 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T22:45:33,527 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:45:33,533 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:33,604 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store 2024-11-23T22:45:33,630 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T22:45:33,635 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:33,637 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:45:33,638 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:33,638 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:33,640 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:45:33,641 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:33,641 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:33,642 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732401933637Disabling compacts and flushes for region at 1732401933637Disabling writes for close at 1732401933641 (+4 ms)Writing region close event to WAL at 1732401933641Closed at 1732401933641 2024-11-23T22:45:33,643 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/.initializing 2024-11-23T22:45:33,643 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:33,657 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:33,670 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C42801%2C1732401931984, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,42801,1732401931984, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/oldWALs, maxLogs=10 2024-11-23T22:45:33,708 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,42801,1732401931984/dd96e8dc8369%2C42801%2C1732401931984.1732401933675 2024-11-23T22:45:33,709 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:33,710 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:33,711 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:33,716 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,717 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T22:45:33,832 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:33,834 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:33,835 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T22:45:33,843 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:33,843 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:33,844 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,854 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T22:45:33,855 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:33,855 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:33,856 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,869 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T22:45:33,870 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:33,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:33,871 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,875 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,876 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,880 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,881 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,885 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:33,888 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:33,897 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:33,898 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63971585, jitterRate=-0.04674910008907318}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:33,910 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732401933730Initializing all the Stores at 1732401933731 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401933732 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401933733 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401933733Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401933733Cleaning up temporary data from old regions at 1732401933881 (+148 ms)Region opened successfully at 1732401933910 (+29 ms) 2024-11-23T22:45:33,916 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T22:45:33,987 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e2b8aad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:34,033 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T22:45:34,050 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T22:45:34,050 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T22:45:34,055 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T22:45:34,056 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T22:45:34,064 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-23T22:45:34,064 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T22:45:34,103 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T22:45:34,118 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T22:45:34,127 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T22:45:34,132 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T22:45:34,135 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T22:45:34,148 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T22:45:34,151 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T22:45:34,157 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T22:45:34,169 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T22:45:34,177 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T22:45:34,190 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T22:45:34,212 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T22:45:34,221 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T22:45:34,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:34,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:34,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,246 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,42801,1732401931984, sessionid=0x10169dfa1350000, setting cluster-up flag (Was=false) 2024-11-23T22:45:34,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,327 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T22:45:34,330 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:34,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,348 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:34,379 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T22:45:34,382 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:34,388 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T22:45:34,429 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(746): ClusterId : c5ac0603-26c7-4672-8afc-f831b0d0ab7b 2024-11-23T22:45:34,429 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(746): ClusterId : c5ac0603-26c7-4672-8afc-f831b0d0ab7b 2024-11-23T22:45:34,431 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:45:34,431 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:45:34,445 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:45:34,445 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:45:34,445 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:45:34,445 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:45:34,454 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:45:34,454 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:45:34,455 DEBUG [RS:1;dd96e8dc8369:44275 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b1f31b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:34,455 DEBUG [RS:0;dd96e8dc8369:33605 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef7c2d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:34,458 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:34,467 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T22:45:34,470 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:33605 2024-11-23T22:45:34,470 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd96e8dc8369:44275 2024-11-23T22:45:34,474 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:45:34,474 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:45:34,474 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T22:45:34,475 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:45:34,475 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:45:34,475 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:45:34,475 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:45:34,478 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,42801,1732401931984 with port=44275, startcode=1732401933167 2024-11-23T22:45:34,478 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,42801,1732401931984 with port=33605, startcode=1732401933016 2024-11-23T22:45:34,483 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,42801,1732401931984 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T22:45:34,492 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:34,492 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:34,493 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:34,493 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:34,493 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10 2024-11-23T22:45:34,494 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,494 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:34,494 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,497 DEBUG [RS:1;dd96e8dc8369:44275 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:45:34,497 DEBUG [RS:0;dd96e8dc8369:33605 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:45:34,501 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:34,502 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:45:34,502 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732401964502 2024-11-23T22:45:34,504 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T22:45:34,506 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T22:45:34,508 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:34,509 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:45:34,509 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T22:45:34,510 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T22:45:34,510 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T22:45:34,510 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T22:45:34,511 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,514 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T22:45:34,516 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T22:45:34,517 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T22:45:34,519 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T22:45:34,519 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T22:45:34,522 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401934521,5,FailOnTimeoutGroup] 2024-11-23T22:45:34,528 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401934522,5,FailOnTimeoutGroup] 2024-11-23T22:45:34,528 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,529 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T22:45:34,531 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,531 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,551 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:45:34,552 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:34,560 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:34,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T22:45:34,565 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:45:34,565 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:34,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:34,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T22:45:34,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:45:34,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:34,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:34,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T22:45:34,573 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52293, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:45:34,573 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:45:34,576 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:45:34,577 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:34,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:34,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T22:45:34,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:45:34,581 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:34,581 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:34,582 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:45:34,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:34,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:34,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:45:34,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:45:34,584 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,584 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:34,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:45:34,592 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:34,592 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68728035, jitterRate=0.024127528071403503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:34,596 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,596 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42801 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732401934560Initializing all the Stores at 1732401934561 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401934561Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401934561Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401934561Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401934561Cleaning up temporary data from old regions at 1732401934584 (+23 ms)Region opened successfully at 1732401934598 (+14 ms) 2024-11-23T22:45:34,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:45:34,599 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:45:34,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:45:34,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:45:34,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:45:34,600 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:34,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732401934598Disabling compacts and flushes for region at 1732401934598Disabling writes for close at 1732401934599 (+1 ms)Writing region close event to WAL at 1732401934600 (+1 ms)Closed at 1732401934600 2024-11-23T22:45:34,602 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:34,602 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:34,602 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T22:45:34,602 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T22:45:34,602 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:45:34,602 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:45:34,604 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:34,604 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T22:45:34,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:34,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T22:45:34,622 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:45:34,624 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T22:45:34,687 DEBUG [RS:1;dd96e8dc8369:44275 {}] zookeeper.ZKUtil(111): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,687 WARN [RS:1;dd96e8dc8369:44275 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:34,687 DEBUG [RS:0;dd96e8dc8369:33605 {}] zookeeper.ZKUtil(111): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,687 INFO [RS:1;dd96e8dc8369:44275 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:34,687 WARN [RS:0;dd96e8dc8369:33605 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:34,687 DEBUG [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,688 INFO [RS:0;dd96e8dc8369:33605 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:34,688 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,689 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,44275,1732401933167] 2024-11-23T22:45:34,689 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,33605,1732401933016] 2024-11-23T22:45:34,712 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:45:34,712 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:45:34,731 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:45:34,731 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:45:34,736 INFO [RS:1;dd96e8dc8369:44275 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:45:34,736 INFO [RS:0;dd96e8dc8369:33605 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:45:34,736 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,736 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,737 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:45:34,737 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:45:34,742 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:45:34,742 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:45:34,743 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,743 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,744 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:34,745 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,745 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,746 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,746 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,746 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:34,746 DEBUG [RS:0;dd96e8dc8369:33605 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:34,746 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:34,746 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:34,746 DEBUG [RS:1;dd96e8dc8369:44275 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,747 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33605,1732401933016-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:34,748 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,748 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,748 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,44275,1732401933167-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:34,771 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:45:34,771 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:45:34,773 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,44275,1732401933167-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,773 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,33605,1732401933016-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,774 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,774 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,774 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.Replication(171): dd96e8dc8369,44275,1732401933167 started 2024-11-23T22:45:34,774 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.Replication(171): dd96e8dc8369,33605,1732401933016 started 2024-11-23T22:45:34,776 WARN [dd96e8dc8369:42801 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T22:45:34,801 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,801 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:34,801 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,44275,1732401933167, RpcServer on dd96e8dc8369/172.17.0.2:44275, sessionid=0x10169dfa1350002 2024-11-23T22:45:34,801 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,33605,1732401933016, RpcServer on dd96e8dc8369/172.17.0.2:33605, sessionid=0x10169dfa1350001 2024-11-23T22:45:34,802 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:45:34,802 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:45:34,802 DEBUG [RS:1;dd96e8dc8369:44275 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,802 DEBUG [RS:0;dd96e8dc8369:33605 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,803 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,44275,1732401933167' 2024-11-23T22:45:34,803 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,33605,1732401933016' 2024-11-23T22:45:34,803 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:45:34,803 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:45:34,804 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:45:34,804 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:45:34,805 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:45:34,805 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:45:34,805 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:45:34,805 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:45:34,805 DEBUG [RS:0;dd96e8dc8369:33605 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:34,805 DEBUG [RS:1;dd96e8dc8369:44275 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:34,805 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,33605,1732401933016' 2024-11-23T22:45:34,805 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,44275,1732401933167' 2024-11-23T22:45:34,805 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:45:34,805 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:45:34,806 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:45:34,806 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:45:34,807 DEBUG [RS:0;dd96e8dc8369:33605 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:45:34,807 INFO [RS:0;dd96e8dc8369:33605 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:45:34,807 INFO [RS:0;dd96e8dc8369:33605 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:45:34,807 DEBUG [RS:1;dd96e8dc8369:44275 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:45:34,807 INFO [RS:1;dd96e8dc8369:44275 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:45:34,807 INFO [RS:1;dd96e8dc8369:44275 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:45:34,912 INFO [RS:1;dd96e8dc8369:44275 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:34,912 INFO [RS:0;dd96e8dc8369:33605 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:34,913 INFO [RS:1;dd96e8dc8369:44275 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C44275%2C1732401933167, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,44275,1732401933167, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:34,913 INFO [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C33605%2C1732401933016, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,33605,1732401933016, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:34,933 INFO [RS:1;dd96e8dc8369:44275 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,44275,1732401933167/dd96e8dc8369%2C44275%2C1732401933167.1732401934916 2024-11-23T22:45:34,933 INFO [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,33605,1732401933016/dd96e8dc8369%2C33605%2C1732401933016.1732401934916 2024-11-23T22:45:34,933 DEBUG [RS:1;dd96e8dc8369:44275 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:34,933 DEBUG [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:35,032 DEBUG [dd96e8dc8369:42801 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T22:45:35,044 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:45:35,047 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:45:35,047 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:45:35,047 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:45:35,047 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:45:35,047 INFO [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:45:35,047 INFO [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:45:35,047 DEBUG [dd96e8dc8369:42801 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:45:35,055 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:35,060 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,33605,1732401933016, state=OPENING 2024-11-23T22:45:35,104 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T22:45:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:35,117 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,118 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,120 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:45:35,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,33605,1732401933016}] 2024-11-23T22:45:35,296 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:45:35,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40681, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:45:35,307 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T22:45:35,308 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:35,309 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T22:45:35,310 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C33605%2C1732401933016.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,33605,1732401933016, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:35,328 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,33605,1732401933016/dd96e8dc8369%2C33605%2C1732401933016.meta.1732401935311.meta 2024-11-23T22:45:35,329 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:35,329 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:35,331 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T22:45:35,333 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T22:45:35,337 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T22:45:35,341 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T22:45:35,341 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:35,342 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T22:45:35,342 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T22:45:35,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T22:45:35,344 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:45:35,344 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:35,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:35,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T22:45:35,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:45:35,346 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:35,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:35,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T22:45:35,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:45:35,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:35,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:35,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T22:45:35,348 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:45:35,348 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:35,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:35,348 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:45:35,349 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:35,350 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:35,350 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:45:35,350 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:45:35,350 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:35,351 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:45:35,352 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69600280, jitterRate=0.03712499141693115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:35,352 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T22:45:35,354 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732401935342Writing region info on filesystem at 1732401935343 (+1 ms)Initializing all the Stores at 1732401935343Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401935343Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401935343Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401935343Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401935343Cleaning up temporary data from old regions at 1732401935350 (+7 ms)Running coprocessor post-open hooks at 1732401935352 (+2 ms)Region opened successfully at 1732401935354 (+2 ms) 2024-11-23T22:45:35,359 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732401935287 2024-11-23T22:45:35,369 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T22:45:35,370 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T22:45:35,370 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:35,372 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,33605,1732401933016, state=OPEN 2024-11-23T22:45:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:35,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:35,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,379 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,380 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:35,380 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:35,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T22:45:35,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,33605,1732401933016 in 258 msec 2024-11-23T22:45:35,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T22:45:35,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 775 msec 2024-11-23T22:45:35,391 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:35,391 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T22:45:35,409 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:45:35,411 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,33605,1732401933016, seqNum=-1] 2024-11-23T22:45:35,428 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:35,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:35,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0330 sec 2024-11-23T22:45:35,451 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732401935451, completionTime=-1 2024-11-23T22:45:35,453 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T22:45:35,454 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T22:45:35,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T22:45:35,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732401995485 2024-11-23T22:45:35,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402055485 2024-11-23T22:45:35,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 31 msec 2024-11-23T22:45:35,487 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T22:45:35,493 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,493 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,493 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,495 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:42801, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,495 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,495 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,502 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T22:45:35,523 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.206sec 2024-11-23T22:45:35,525 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T22:45:35,526 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T22:45:35,527 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T22:45:35,527 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T22:45:35,527 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T22:45:35,528 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:35,528 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T22:45:35,529 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T22:45:35,530 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T22:45:35,530 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42801,1732401931984-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:35,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:35,540 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T22:45:35,540 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T22:45:35,543 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,42801,-1 for getting cluster id 2024-11-23T22:45:35,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T22:45:35,553 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c5ac0603-26c7-4672-8afc-f831b0d0ab7b' 2024-11-23T22:45:35,555 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T22:45:35,555 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c5ac0603-26c7-4672-8afc-f831b0d0ab7b" 2024-11-23T22:45:35,555 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319de198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:35,556 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,42801,-1] 2024-11-23T22:45:35,558 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T22:45:35,560 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:35,561 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T22:45:35,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:35,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:45:35,571 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,33605,1732401933016, seqNum=-1] 2024-11-23T22:45:35,571 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:35,574 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:35,609 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:35,617 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T22:45:35,623 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:35,625 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12b12a2a 2024-11-23T22:45:35,626 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T22:45:35,628 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T22:45:35,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732401935610', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:45:35,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 2024-11-23T22:45:35,643 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T22:45:35,646 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732401935610" procId is: 4 2024-11-23T22:45:35,646 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:35,648 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T22:45:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:35,669 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3828761d10f7992e79aa7327b8bb57ea, NAME => 'testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732401935610', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:35,675 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:35,676 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1722): Closing 3828761d10f7992e79aa7327b8bb57ea, disabling compactions & flushes 2024-11-23T22:45:35,676 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:35,676 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:35,676 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. after waiting 0 ms 2024-11-23T22:45:35,676 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:35,676 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:35,676 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732401935610-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3828761d10f7992e79aa7327b8bb57ea: Waiting for close lock at 1732401935676Disabling compacts and flushes for region at 1732401935676Disabling writes for close at 1732401935676Writing region close event to WAL at 1732401935676Closed at 1732401935676 2024-11-23T22:45:35,678 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T22:45:35,683 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732401935678"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732401935678"}]},"ts":"1732401935678"} 2024-11-23T22:45:35,689 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T22:45:35,690 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T22:45:35,693 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732401935610","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732401935690"}]},"ts":"1732401935690"} 2024-11-23T22:45:35,698 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732401935610, state=ENABLING in hbase:meta 2024-11-23T22:45:35,699 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:45:35,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:45:35,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:45:35,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:45:35,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:45:35,700 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:45:35,700 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:45:35,700 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:45:35,702 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, ASSIGN}] 2024-11-23T22:45:35,704 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, ASSIGN 2024-11-23T22:45:35,705 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, ASSIGN; state=OFFLINE, location=dd96e8dc8369,44275,1732401933167; forceNewPlan=false, retain=false 2024-11-23T22:45:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:35,858 INFO [dd96e8dc8369:42801 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T22:45:35,859 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=OPENING, regionLocation=dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:35,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, ASSIGN because future has completed 2024-11-23T22:45:35,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167}] 2024-11-23T22:45:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:36,019 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:45:36,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38683, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:45:36,027 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:36,027 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3828761d10f7992e79aa7327b8bb57ea, NAME => 'testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:36,028 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732401935610 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,028 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:36,029 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,029 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,030 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,034 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3828761d10f7992e79aa7327b8bb57ea columnFamilyName family 2024-11-23T22:45:36,035 DEBUG [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:36,035 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(327): Store=3828761d10f7992e79aa7327b8bb57ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:36,035 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,036 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,037 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,037 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,037 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,038 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,044 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:36,044 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3828761d10f7992e79aa7327b8bb57ea; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71655940, jitterRate=0.06775671243667603}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:45:36,045 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:36,046 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3828761d10f7992e79aa7327b8bb57ea: Running coprocessor pre-open hook at 1732401936029Writing region info on filesystem at 1732401936029Initializing all the Stores at 1732401936030 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401936030Cleaning up temporary data from old regions at 1732401936037 (+7 ms)Running coprocessor post-open hooks at 1732401936045 (+8 ms)Region opened successfully at 1732401936046 (+1 ms) 2024-11-23T22:45:36,047 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea., pid=6, masterSystemTime=1732401936019 2024-11-23T22:45:36,050 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:36,051 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:36,051 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:36,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167 because future has completed 2024-11-23T22:45:36,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T22:45:36,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167 in 191 msec 2024-11-23T22:45:36,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T22:45:36,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, ASSIGN in 356 msec 2024-11-23T22:45:36,061 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T22:45:36,061 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732401935610","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732401936061"}]},"ts":"1732401936061"} 2024-11-23T22:45:36,064 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732401935610, state=ENABLED in hbase:meta 2024-11-23T22:45:36,065 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T22:45:36,067 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732401935610 in 428 msec 2024-11-23T22:45:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:36,281 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732401935610 get assigned. Timeout = 60000ms 2024-11-23T22:45:36,281 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732401935610 completed 2024-11-23T22:45:36,286 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:45:36,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732401935610 assigned to meta. Checking AM states. 2024-11-23T22:45:36,294 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:45:36,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732401935610 assigned. 2024-11-23T22:45:36,303 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732401935610', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea., hostname=dd96e8dc8369,44275,1732401933167, seqNum=2] 2024-11-23T22:45:36,304 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:36,307 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:36,316 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3828761d10f7992e79aa7327b8bb57ea 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T22:45:36,343 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:36,389 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/.tmp/family/eb43968001ae497ea113cd917dfc366d is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T22:45:36,394 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/.tmp/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:36,426 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/.tmp/family/eb43968001ae497ea113cd917dfc366d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:36,427 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, delay=949 ms 2024-11-23T22:45:36,429 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T22:45:36,434 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 3828761d10f7992e79aa7327b8bb57ea in 118ms, sequenceid=6, compaction requested=false 2024-11-23T22:45:36,435 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732401935610' 2024-11-23T22:45:36,437 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3828761d10f7992e79aa7327b8bb57ea: 2024-11-23T22:45:37,381 DEBUG [hfile-prefetch-1732401936427 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key eb43968001ae497ea113cd917dfc366d_0. 2024-11-23T22:45:37,389 DEBUG [hfile-prefetch-1732401936427 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key eb43968001ae497ea113cd917dfc366d_117. 2024-11-23T22:45:37,389 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, with total blocks 2, and data blocks 1 2024-11-23T22:45:37,390 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T22:45:37,390 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T22:45:37,390 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d. Will sleep 100ms and try the verification again. 2024-11-23T22:45:37,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=3828761d10f7992e79aa7327b8bb57ea, source=dd96e8dc8369,44275,1732401933167, destination=dd96e8dc8369,33605,1732401933016, warming up region on dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:37,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=3828761d10f7992e79aa7327b8bb57ea, source=dd96e8dc8369,44275,1732401933167, destination=dd96e8dc8369,33605,1732401933016, running balancer 2024-11-23T22:45:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE 2024-11-23T22:45:37,454 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE 2024-11-23T22:45:37,454 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3828761d10f7992e79aa7327b8bb57ea, NAME => 'testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:37,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:37,456 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=CLOSING, regionLocation=dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:37,456 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,457 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3828761d10f7992e79aa7327b8bb57ea columnFamilyName family 2024-11-23T22:45:37,457 DEBUG [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:37,459 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE because future has completed 2024-11-23T22:45:37,460 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-23T22:45:37,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167}] 2024-11-23T22:45:37,461 DEBUG [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:37,462 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(327): Store=3828761d10f7992e79aa7327b8bb57ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1722): Closing 3828761d10f7992e79aa7327b8bb57ea, disabling compactions & flushes 2024-11-23T22:45:37,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. after waiting 0 ms 2024-11-23T22:45:37,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,463 DEBUG [StoreFileCloser-3828761d10f7992e79aa7327b8bb57ea-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:37,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33605 {}] regionserver.HRegion(1676): Region close journal for 3828761d10f7992e79aa7327b8bb57ea: Waiting for close lock at 1732401937462Disabling compacts and flushes for region at 1732401937462Disabling writes for close at 1732401937462Writing region close event to WAL at 1732401937464 (+2 ms)Closed at 1732401937464 2024-11-23T22:45:37,466 WARN [hfile-prefetch-1732401936427 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-23T22:45:37,466 DEBUG [hfile-prefetch-1732401936427 {}] hfile.PrefetchExecutor(121): Prefetch completed for eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42801 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T22:45:37,608 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T22:45:37,608 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T22:45:37,609 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:37,609 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:37,619 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,619 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-23T22:45:37,620 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3828761d10f7992e79aa7327b8bb57ea, disabling compactions & flushes 2024-11-23T22:45:37,620 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,620 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,621 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. after waiting 0 ms 2024-11-23T22:45:37,621 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,621 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T22:45:37,621 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T22:45:37,622 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:37,622 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:37,622 DEBUG [StoreFileCloser-3828761d10f7992e79aa7327b8bb57ea-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file eb43968001ae497ea113cd917dfc366d, starting offset: 0, end offset: 9223372036854775807 2024-11-23T22:45:37,624 DEBUG [StoreFileCloser-3828761d10f7992e79aa7327b8bb57ea-family-1 {}] bucket.BucketCache(856): removed key eb43968001ae497ea113cd917dfc366d_0 from back map with offset lock 2094080 in the evict process 2024-11-23T22:45:37,631 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T22:45:37,632 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,633 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3828761d10f7992e79aa7327b8bb57ea: Waiting for close lock at 1732401937620Running coprocessor pre-close hooks at 1732401937620Disabling compacts and flushes for region at 1732401937620Disabling writes for close at 1732401937621 (+1 ms)Writing region close event to WAL at 1732401937625 (+4 ms)Running coprocessor post-close hooks at 1732401937631 (+6 ms)Closed at 1732401937632 (+1 ms) 2024-11-23T22:45:37,633 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3828761d10f7992e79aa7327b8bb57ea move to dd96e8dc8369,33605,1732401933016 record at close sequenceid=6 2024-11-23T22:45:37,637 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,637 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=CLOSED 2024-11-23T22:45:37,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167 because future has completed 2024-11-23T22:45:37,642 ERROR [bucket-cache-persister {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:48) ~[classes/:?] 2024-11-23T22:45:37,642 WARN [Thread-91 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401937608 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401937622 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T22:45:37,644 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T22:45:37,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T22:45:37,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,44275,1732401933167 in 181 msec 2024-11-23T22:45:37,646 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE; state=CLOSED, location=dd96e8dc8369,33605,1732401933016; forceNewPlan=false, retain=false 2024-11-23T22:45:37,797 INFO [dd96e8dc8369:42801 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T22:45:37,798 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=OPENING, regionLocation=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:37,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE because future has completed 2024-11-23T22:45:37,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,33605,1732401933016}] 2024-11-23T22:45:37,960 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,960 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3828761d10f7992e79aa7327b8bb57ea, NAME => 'testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:37,961 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732401935610 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,961 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:37,961 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,961 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,962 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,963 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3828761d10f7992e79aa7327b8bb57ea columnFamilyName family 2024-11-23T22:45:37,964 DEBUG [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:37,965 DEBUG [StoreFileOpener-3828761d10f7992e79aa7327b8bb57ea-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d, delay=935 ms 2024-11-23T22:45:37,967 DEBUG [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:37,967 INFO [StoreOpener-3828761d10f7992e79aa7327b8bb57ea-1 {}] regionserver.HStore(327): Store=3828761d10f7992e79aa7327b8bb57ea/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:37,967 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,968 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,968 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,968 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,968 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,973 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,973 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3828761d10f7992e79aa7327b8bb57ea; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72877321, jitterRate=0.08595670759677887}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:45:37,973 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:37,974 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3828761d10f7992e79aa7327b8bb57ea: Running coprocessor pre-open hook at 1732401937961Writing region info on filesystem at 1732401937961Initializing all the Stores at 1732401937962 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401937962Cleaning up temporary data from old regions at 1732401937968 (+6 ms)Running coprocessor post-open hooks at 1732401937973 (+5 ms)Region opened successfully at 1732401937974 (+1 ms) 2024-11-23T22:45:37,975 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea., pid=9, masterSystemTime=1732401937957 2024-11-23T22:45:37,977 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,978 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:37,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3828761d10f7992e79aa7327b8bb57ea, regionState=OPEN, openSeqNum=10, regionLocation=dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:37,981 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,33605,1732401933016 because future has completed 2024-11-23T22:45:37,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T22:45:37,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3828761d10f7992e79aa7327b8bb57ea, server=dd96e8dc8369,33605,1732401933016 in 178 msec 2024-11-23T22:45:37,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732401935610, region=3828761d10f7992e79aa7327b8bb57ea, REOPEN/MOVE in 531 msec 2024-11-23T22:45:38,006 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T22:45:38,007 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:45:38,007 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:38,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,013 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T22:45:38,013 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T22:45:38,013 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-23T22:45:38,014 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,42801,1732401931984 2024-11-23T22:45:38,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:38,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:38,030 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:38,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:38,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:38,031 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:38,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,032 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:45:38,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,034 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:45:38,034 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:38,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,035 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,33605,1732401933016' ***** 2024-11-23T22:45:38,035 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:45:38,035 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,44275,1732401933167' ***** 2024-11-23T22:45:38,035 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:45:38,035 INFO [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T22:45:38,035 INFO [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T22:45:38,036 INFO [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence 2024-11-23T22:45:38,036 INFO [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/bucket.persistence 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T22:45:38,036 INFO [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T22:45:38,037 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T22:45:38,037 INFO [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T22:45:38,037 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T22:45:38,047 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1487): Thread RS:1;dd96e8dc8369:44275 started persisting bucket cache to file 2024-11-23T22:45:38,047 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:38,047 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:38,047 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1487): Thread RS:0;dd96e8dc8369:33605 started persisting bucket cache to file 2024-11-23T22:45:38,048 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:38,048 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:38,051 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:38,051 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:38,051 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401938047 2024-11-23T22:45:38,051 WARN [Thread-95 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-23T22:45:38,051 DEBUG [RS:1;dd96e8dc8369:44275 {}] bucket.BucketCache(1509): Thread RS:1;dd96e8dc8369:44275 finished persisting bucket cache to file, renaming 2024-11-23T22:45:38,051 ERROR [RS:0;dd96e8dc8369:33605 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:45:38,052 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:38,052 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401938047 2024-11-23T22:45:38,052 DEBUG [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1509): Thread RS:0;dd96e8dc8369:33605 finished persisting bucket cache to file, renaming 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:45:38,052 WARN [RS:0;dd96e8dc8369:33605 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:45:38,052 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:45:38,052 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:38,052 INFO [RS:0;dd96e8dc8369:33605 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:45:38,052 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:38,052 INFO [RS:0;dd96e8dc8369:33605 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:45:38,052 INFO [RS:1;dd96e8dc8369:44275 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd96e8dc8369:44275. 2024-11-23T22:45:38,052 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(3091): Received CLOSE for 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:38,052 DEBUG [RS:1;dd96e8dc8369:44275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:38,052 DEBUG [RS:1;dd96e8dc8369:44275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,053 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,44275,1732401933167; all regions closed. 2024-11-23T22:45:38,053 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:38,053 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:38,053 INFO [RS:0;dd96e8dc8369:33605 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:33605. 2024-11-23T22:45:38,053 DEBUG [RS:0;dd96e8dc8369:33605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:38,053 DEBUG [RS:0;dd96e8dc8369:33605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,053 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T22:45:38,054 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:45:38,054 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:45:38,054 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:45:38,054 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T22:45:38,054 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3828761d10f7992e79aa7327b8bb57ea, disabling compactions & flushes 2024-11-23T22:45:38,054 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:38,054 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:38,054 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. after waiting 0 ms 2024-11-23T22:45:38,054 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:38,054 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T22:45:38,054 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3828761d10f7992e79aa7327b8bb57ea=testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea.} 2024-11-23T22:45:38,054 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T22:45:38,055 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:45:38,055 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:45:38,055 DEBUG [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3828761d10f7992e79aa7327b8bb57ea 2024-11-23T22:45:38,055 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:45:38,055 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:45:38,055 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:38,055 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:45:38,055 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-23T22:45:38,058 DEBUG [StoreFileCloser-3828761d10f7992e79aa7327b8bb57ea-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/family/eb43968001ae497ea113cd917dfc366d 2024-11-23T22:45:38,060 DEBUG [RS:1;dd96e8dc8369:44275 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:38,060 INFO [RS:1;dd96e8dc8369:44275 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C44275%2C1732401933167:(num 1732401934916) 2024-11-23T22:45:38,061 DEBUG [RS:1;dd96e8dc8369:44275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,061 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:38,061 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:38,061 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:38,061 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:45:38,062 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:45:38,062 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:45:38,062 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:38,062 INFO [RS:1;dd96e8dc8369:44275 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44275 2024-11-23T22:45:38,063 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:38,069 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnRegionMove1732401935610/3828761d10f7992e79aa7327b8bb57ea/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T22:45:38,069 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:38,069 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3828761d10f7992e79aa7327b8bb57ea: Waiting for close lock at 1732401938053Running coprocessor pre-close hooks at 1732401938054 (+1 ms)Disabling compacts and flushes for region at 1732401938054Disabling writes for close at 1732401938054Writing region close event to WAL at 1732401938058 (+4 ms)Running coprocessor post-close hooks at 1732401938069 (+11 ms)Closed at 1732401938069 2024-11-23T22:45:38,069 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea. 2024-11-23T22:45:38,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:38,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,44275,1732401933167 2024-11-23T22:45:38,074 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:38,076 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,44275,1732401933167] 2024-11-23T22:45:38,079 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e5f44b084de14f17a202a6fa015bc4a2 2024-11-23T22:45:38,095 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,44275,1732401933167 already deleted, retry=false 2024-11-23T22:45:38,096 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,44275,1732401933167 expired; onlineServers=1 2024-11-23T22:45:38,103 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/e5f44b084de14f17a202a6fa015bc4a2 is 205, key is testBlockEvictionOnRegionMove1732401935610,,1732401935629.3828761d10f7992e79aa7327b8bb57ea./info:regioninfo/1732401937978/Put/seqid=0 2024-11-23T22:45:38,105 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/e5f44b084de14f17a202a6fa015bc4a2 2024-11-23T22:45:38,119 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83ef72e710f2417ea8e50700c7c69def 2024-11-23T22:45:38,139 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/83ef72e710f2417ea8e50700c7c69def is 43, key is default/ns:d/1732401935436/Put/seqid=0 2024-11-23T22:45:38,140 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/83ef72e710f2417ea8e50700c7c69def 2024-11-23T22:45:38,154 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1e87e320fbdb46b6ba2712f330d1192d 2024-11-23T22:45:38,168 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/1e87e320fbdb46b6ba2712f330d1192d is 78, key is testBlockEvictionOnRegionMove1732401935610/table:state/1732401936061/Put/seqid=0 2024-11-23T22:45:38,169 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/1e87e320fbdb46b6ba2712f330d1192d 2024-11-23T22:45:38,174 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/e5f44b084de14f17a202a6fa015bc4a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/e5f44b084de14f17a202a6fa015bc4a2 2024-11-23T22:45:38,176 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/e5f44b084de14f17a202a6fa015bc4a2, delay=906 ms 2024-11-23T22:45:38,178 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/e5f44b084de14f17a202a6fa015bc4a2, entries=15, sequenceid=15, filesize=7.6 K 2024-11-23T22:45:38,179 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/83ef72e710f2417ea8e50700c7c69def as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/83ef72e710f2417ea8e50700c7c69def 2024-11-23T22:45:38,181 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/83ef72e710f2417ea8e50700c7c69def, delay=997 ms 2024-11-23T22:45:38,183 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/83ef72e710f2417ea8e50700c7c69def, entries=2, sequenceid=15, filesize=5.0 K 2024-11-23T22:45:38,184 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/1e87e320fbdb46b6ba2712f330d1192d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/1e87e320fbdb46b6ba2712f330d1192d 2024-11-23T22:45:38,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44275-0x10169dfa1350002, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,185 INFO [RS:1;dd96e8dc8369:44275 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:38,185 INFO [RS:1;dd96e8dc8369:44275 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,44275,1732401933167; zookeeper connection closed. 2024-11-23T22:45:38,186 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/1e87e320fbdb46b6ba2712f330d1192d, delay=927 ms 2024-11-23T22:45:38,186 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5b0c6c63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5b0c6c63 2024-11-23T22:45:38,188 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/1e87e320fbdb46b6ba2712f330d1192d, entries=2, sequenceid=15, filesize=5.3 K 2024-11-23T22:45:38,189 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=15, compaction requested=false 2024-11-23T22:45:38,189 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T22:45:38,191 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/e5f44b084de14f17a202a6fa015bc4a2 2024-11-23T22:45:38,192 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/83ef72e710f2417ea8e50700c7c69def 2024-11-23T22:45:38,193 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/1e87e320fbdb46b6ba2712f330d1192d 2024-11-23T22:45:38,198 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T22:45:38,199 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T22:45:38,199 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:38,200 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732401938054Running coprocessor pre-close hooks at 1732401938054Disabling compacts and flushes for region at 1732401938055 (+1 ms)Disabling writes for close at 1732401938055Obtaining lock to block concurrent updates at 1732401938055Preparing flush snapshotting stores in 1588230740 at 1732401938055Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732401938056 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732401938057 (+1 ms)Flushing 1588230740/info: creating writer at 1732401938057Flushing 1588230740/info: appending metadata at 1732401938102 (+45 ms)Flushing 1588230740/info: closing flushed file at 1732401938102Flushing 1588230740/ns: creating writer at 1732401938109 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732401938138 (+29 ms)Flushing 1588230740/ns: closing flushed file at 1732401938138Flushing 1588230740/table: creating writer at 1732401938145 (+7 ms)Flushing 1588230740/table: appending metadata at 1732401938167 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732401938167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@288ffbff: reopening flushed file at 1732401938173 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11c6ea3d: reopening flushed file at 1732401938178 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b2dee78: reopening flushed file at 1732401938183 (+5 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 134ms, sequenceid=15, compaction requested=false at 1732401938189 (+6 ms)Writing region close event to WAL at 1732401938193 (+4 ms)Running coprocessor post-close hooks at 1732401938198 (+5 ms)Closed at 1732401938199 (+1 ms) 2024-11-23T22:45:38,200 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:38,255 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,33605,1732401933016; all regions closed. 2024-11-23T22:45:38,258 DEBUG [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:38,258 INFO [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C33605%2C1732401933016.meta:.meta(num 1732401935311) 2024-11-23T22:45:38,261 DEBUG [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:38,261 INFO [RS:0;dd96e8dc8369:33605 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C33605%2C1732401933016:(num 1732401934916) 2024-11-23T22:45:38,261 DEBUG [RS:0;dd96e8dc8369:33605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:38,261 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:38,262 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:38,262 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:38,262 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:38,262 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:38,262 INFO [RS:0;dd96e8dc8369:33605 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33605 2024-11-23T22:45:38,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,33605,1732401933016 2024-11-23T22:45:38,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:38,274 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:38,275 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,33605,1732401933016] 2024-11-23T22:45:38,295 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,33605,1732401933016 already deleted, retry=false 2024-11-23T22:45:38,295 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,33605,1732401933016 expired; onlineServers=0 2024-11-23T22:45:38,295 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,42801,1732401931984' ***** 2024-11-23T22:45:38,295 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T22:45:38,296 INFO [M:0;dd96e8dc8369:42801 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:38,296 INFO [M:0;dd96e8dc8369:42801 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:38,296 DEBUG [M:0;dd96e8dc8369:42801 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T22:45:38,297 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T22:45:38,297 DEBUG [M:0;dd96e8dc8369:42801 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T22:45:38,297 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401934521 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401934521,5,FailOnTimeoutGroup] 2024-11-23T22:45:38,297 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401934522 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401934522,5,FailOnTimeoutGroup] 2024-11-23T22:45:38,297 INFO [M:0;dd96e8dc8369:42801 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:38,297 INFO [M:0;dd96e8dc8369:42801 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:38,297 DEBUG [M:0;dd96e8dc8369:42801 {}] master.HMaster(1795): Stopping service threads 2024-11-23T22:45:38,297 INFO [M:0;dd96e8dc8369:42801 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T22:45:38,298 INFO [M:0;dd96e8dc8369:42801 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:45:38,298 INFO [M:0;dd96e8dc8369:42801 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T22:45:38,300 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T22:45:38,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:38,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:38,326 DEBUG [M:0;dd96e8dc8369:42801 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-23T22:45:38,327 DEBUG [M:0;dd96e8dc8369:42801 {}] master.ActiveMasterManager(353): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-23T22:45:38,327 INFO [M:0;dd96e8dc8369:42801 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.lastflushedseqids 2024-11-23T22:45:38,370 INFO [M:0;dd96e8dc8369:42801 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T22:45:38,370 INFO [M:0;dd96e8dc8369:42801 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T22:45:38,370 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:45:38,370 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:38,370 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:38,371 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:45:38,371 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:38,371 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-23T22:45:38,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,385 INFO [RS:0;dd96e8dc8369:33605 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:38,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33605-0x10169dfa1350001, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,385 INFO [RS:0;dd96e8dc8369:33605 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,33605,1732401933016; zookeeper connection closed. 2024-11-23T22:45:38,385 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@792da874 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@792da874 2024-11-23T22:45:38,386 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-23T22:45:38,388 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8a6caccd50064007b1ec434c1de66502 2024-11-23T22:45:38,403 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6caccd50064007b1ec434c1de66502 is 82, key is hbase:meta,,1/info:regioninfo/1732401935370/Put/seqid=0 2024-11-23T22:45:38,404 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6caccd50064007b1ec434c1de66502 2024-11-23T22:45:38,412 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6902347735d244c2a576853cf7667673 2024-11-23T22:45:38,430 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6902347735d244c2a576853cf7667673 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732401936067/Put/seqid=0 2024-11-23T22:45:38,431 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6902347735d244c2a576853cf7667673 2024-11-23T22:45:38,439 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 05d6015884624b4c830eba16bae3e65e 2024-11-23T22:45:38,452 DEBUG [M:0;dd96e8dc8369:42801 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05d6015884624b4c830eba16bae3e65e is 69, key is dd96e8dc8369,33605,1732401933016/rs:state/1732401934587/Put/seqid=0 2024-11-23T22:45:38,453 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05d6015884624b4c830eba16bae3e65e 2024-11-23T22:45:38,454 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/8a6caccd50064007b1ec434c1de66502 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a6caccd50064007b1ec434c1de66502 2024-11-23T22:45:38,456 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/8a6caccd50064007b1ec434c1de66502, entries=8, sequenceid=84, filesize=5.5 K 2024-11-23T22:45:38,457 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6902347735d244c2a576853cf7667673 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6902347735d244c2a576853cf7667673 2024-11-23T22:45:38,460 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6902347735d244c2a576853cf7667673, entries=9, sequenceid=84, filesize=6.5 K 2024-11-23T22:45:38,460 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/05d6015884624b4c830eba16bae3e65e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05d6015884624b4c830eba16bae3e65e 2024-11-23T22:45:38,462 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/05d6015884624b4c830eba16bae3e65e, entries=2, sequenceid=84, filesize=5.1 K 2024-11-23T22:45:38,464 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false 2024-11-23T22:45:38,466 INFO [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:38,466 DEBUG [M:0;dd96e8dc8369:42801 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732401938370Disabling compacts and flushes for region at 1732401938370Disabling writes for close at 1732401938371 (+1 ms)Obtaining lock to block concurrent updates at 1732401938371Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732401938371Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732401938373 (+2 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732401938374 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732401938374Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732401938402 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732401938402Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732401938406 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732401938430 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732401938430Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732401938434 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732401938452 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732401938452Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da7d62b: reopening flushed file at 1732401938454 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d8bcbb1: reopening flushed file at 1732401938457 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe5f1ef: reopening flushed file at 1732401938460 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=84, compaction requested=false at 1732401938464 (+4 ms)Writing region close event to WAL at 1732401938466 (+2 ms)Closed at 1732401938466 2024-11-23T22:45:38,468 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:38,468 INFO [M:0;dd96e8dc8369:42801 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T22:45:38,468 INFO [M:0;dd96e8dc8369:42801 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42801 2024-11-23T22:45:38,468 INFO [M:0;dd96e8dc8369:42801 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:38,574 INFO [M:0;dd96e8dc8369:42801 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:38,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42801-0x10169dfa1350000, quorum=127.0.0.1:57408, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:38,583 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T22:45:38,603 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T22:45:38,603 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/dd96e8dc8369:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/dd96e8dc8369:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732401936427 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732401938181 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/dd96e8dc8369:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/dd96e8dc8369:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732401938176 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732401937965 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=457 (was 453) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7214 (was 7633) 2024-11-23T22:45:38,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=457, ProcessCount=11, AvailableMemoryMB=7213 2024-11-23T22:45:38,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/hadoop-log-dir so I do NOT create it in target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08 2024-11-23T22:45:38,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/hadoop-tmp-dir so I do NOT create it in target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08 2024-11-23T22:45:38,617 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,619 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/cluster_41ec5fdb-0530-873b-c6cf-fa05e973fe77, deleteOnExit=true 2024-11-23T22:45:38,622 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/cluster_41ec5fdb-0530-873b-c6cf-fa05e973fe77/zookeeper_0, clientPort=51250, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/cluster_41ec5fdb-0530-873b-c6cf-fa05e973fe77/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/cluster_41ec5fdb-0530-873b-c6cf-fa05e973fe77/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T22:45:38,623 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51250 2024-11-23T22:45:38,633 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 with version=8 2024-11-23T22:45:38,633 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase-staging 2024-11-23T22:45:38,636 INFO [Time-limited test {}] client.ConnectionUtils(128): master/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:38,637 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T22:45:38,637 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:38,638 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39879 2024-11-23T22:45:38,640 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39879 connecting to ZooKeeper ensemble=127.0.0.1:51250 2024-11-23T22:45:38,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:398790x0, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:38,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39879-0x10169dfbc690000 connected 2024-11-23T22:45:38,770 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,770 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,771 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,772 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9, hbase.cluster.distributed=false 2024-11-23T22:45:38,775 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:38,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39879 2024-11-23T22:45:38,792 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39879 2024-11-23T22:45:38,794 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39879 2024-11-23T22:45:38,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39879 2024-11-23T22:45:38,795 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39879 2024-11-23T22:45:38,819 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:38,819 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:45:38,820 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:38,822 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43635 2024-11-23T22:45:38,824 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43635 connecting to ZooKeeper ensemble=127.0.0.1:51250 2024-11-23T22:45:38,825 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,825 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436350x0, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:38,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:436350x0, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,834 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:45:38,835 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43635-0x10169dfbc690001 connected 2024-11-23T22:45:38,837 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache 2024-11-23T22:45:38,837 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T22:45:38,837 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T22:45:38,837 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T22:45:38,837 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T22:45:38,837 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T22:45:38,838 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T22:45:38,838 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence is complete. 2024-11-23T22:45:38,840 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T22:45:38,842 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:45:38,843 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:45:38,844 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:38,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-23T22:45:38,847 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43635 2024-11-23T22:45:38,848 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43635 2024-11-23T22:45:38,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-23T22:45:38,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43635 2024-11-23T22:45:38,876 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:38,876 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:45:38,877 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:38,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37423 2024-11-23T22:45:38,882 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37423 connecting to ZooKeeper ensemble=127.0.0.1:51250 2024-11-23T22:45:38,882 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,883 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:38,896 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:374230x0, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:38,897 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:374230x0, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:38,898 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:45:38,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37423-0x10169dfbc690002 connected 2024-11-23T22:45:38,910 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache 2024-11-23T22:45:38,910 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T22:45:38,910 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T22:45:38,911 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T22:45:38,912 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T22:45:38,913 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T22:45:38,913 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T22:45:38,914 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T22:45:38,914 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence is complete. 2024-11-23T22:45:38,916 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:45:38,918 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T22:45:38,921 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:38,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37423 2024-11-23T22:45:38,945 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37423 2024-11-23T22:45:38,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37423 2024-11-23T22:45:38,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37423 2024-11-23T22:45:38,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37423 2024-11-23T22:45:38,978 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;dd96e8dc8369:39879 2024-11-23T22:45:38,978 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:38,990 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:38,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:38,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:38,992 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:39,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,001 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:39,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,001 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,002 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T22:45:39,003 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/dd96e8dc8369,39879,1732401938635 from backup master directory 2024-11-23T22:45:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,011 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:39,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:39,012 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T22:45:39,012 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:39,012 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,014 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase.id] with ID: c6803948-0b9e-4609-825f-8b193152163e 2024-11-23T22:45:39,014 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.tmp/hbase.id 2024-11-23T22:45:39,021 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/hbase.id] 2024-11-23T22:45:39,083 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:39,084 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T22:45:39,086 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-23T22:45:39,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,095 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,100 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:45:39,101 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T22:45:39,102 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:39,122 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store 2024-11-23T22:45:39,128 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:45:39,129 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:39,129 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732401939129Disabling compacts and flushes for region at 1732401939129Disabling writes for close at 1732401939129Writing region close event to WAL at 1732401939129Closed at 1732401939129 2024-11-23T22:45:39,129 WARN [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/.initializing 2024-11-23T22:45:39,129 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,134 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:39,138 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=dd96e8dc8369%2C39879%2C1732401938635, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,39879,1732401938635, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/oldWALs, maxLogs=10 2024-11-23T22:45:39,159 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/WALs/dd96e8dc8369,39879,1732401938635/dd96e8dc8369%2C39879%2C1732401938635.1732401939139 2024-11-23T22:45:39,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:39,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:39,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:39,159 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,160 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T22:45:39,165 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:39,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T22:45:39,169 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:39,169 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T22:45:39,172 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:39,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T22:45:39,175 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:39,176 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,176 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,176 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,176 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,176 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,177 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:39,177 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T22:45:39,182 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:39,183 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75063734, jitterRate=0.1185368001461029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:39,184 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732401939160Initializing all the Stores at 1732401939160Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401939160Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401939160Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401939160Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401939160Cleaning up temporary data from old regions at 1732401939176 (+16 ms)Region opened successfully at 1732401939183 (+7 ms) 2024-11-23T22:45:39,184 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T22:45:39,188 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c96c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:39,189 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T22:45:39,189 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T22:45:39,190 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T22:45:39,190 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T22:45:39,190 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T22:45:39,191 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T22:45:39,191 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T22:45:39,193 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T22:45:39,194 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T22:45:39,200 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T22:45:39,201 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T22:45:39,202 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T22:45:39,211 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T22:45:39,212 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T22:45:39,213 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T22:45:39,221 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T22:45:39,223 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T22:45:39,232 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T22:45:39,235 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T22:45:39,242 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T22:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:39,253 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,253 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,255 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=dd96e8dc8369,39879,1732401938635, sessionid=0x10169dfbc690000, setting cluster-up flag (Was=false) 2024-11-23T22:45:39,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,274 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,380 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T22:45:39,386 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,412 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:39,453 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T22:45:39,455 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:39,456 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T22:45:39,459 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:39,459 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T22:45:39,459 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T22:45:39,460 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: dd96e8dc8369,39879,1732401938635 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T22:45:39,462 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(746): ClusterId : c6803948-0b9e-4609-825f-8b193152163e 2024-11-23T22:45:39,462 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:45:39,462 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:39,462 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=5, maxPoolSize=5 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/dd96e8dc8369:0, corePoolSize=10, maxPoolSize=10 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:39,463 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,473 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732401969473 2024-11-23T22:45:39,473 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T22:45:39,473 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T22:45:39,474 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T22:45:39,475 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T22:45:39,475 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(746): ClusterId : c6803948-0b9e-4609-825f-8b193152163e 2024-11-23T22:45:39,475 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:45:39,475 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:39,475 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T22:45:39,475 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T22:45:39,475 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T22:45:39,476 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,476 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,477 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T22:45:39,481 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T22:45:39,481 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T22:45:39,482 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T22:45:39,483 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T22:45:39,483 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T22:45:39,483 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401939483,5,FailOnTimeoutGroup] 2024-11-23T22:45:39,484 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401939483,5,FailOnTimeoutGroup] 2024-11-23T22:45:39,484 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,485 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:45:39,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T22:45:39,485 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:45:39,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,485 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,495 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:45:39,496 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:45:39,505 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T22:45:39,505 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:39,506 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:45:39,507 DEBUG [RS:0;dd96e8dc8369:43635 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1774d019, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:39,508 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:45:39,509 DEBUG [RS:1;dd96e8dc8369:37423 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291c4b66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:39,521 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;dd96e8dc8369:43635 2024-11-23T22:45:39,521 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:45:39,521 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:45:39,521 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:45:39,522 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,39879,1732401938635 with port=43635, startcode=1732401938818 2024-11-23T22:45:39,522 DEBUG [RS:0;dd96e8dc8369:43635 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:45:39,527 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;dd96e8dc8369:37423 2024-11-23T22:45:39,527 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:45:39,527 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:45:39,527 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:45:39,529 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,39879,1732401938635 with port=37423, startcode=1732401938876 2024-11-23T22:45:39,530 DEBUG [RS:1;dd96e8dc8369:37423 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:45:39,533 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57313, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:45:39,534 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,534 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,537 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:39,537 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T22:45:39,537 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:45:39,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:39,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:39,549 DEBUG [RS:0;dd96e8dc8369:43635 {}] zookeeper.ZKUtil(111): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,549 WARN [RS:0;dd96e8dc8369:43635 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:39,550 INFO [RS:0;dd96e8dc8369:43635 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:39,550 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T22:45:39,550 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,552 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49781, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:45:39,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,554 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,556 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:39,557 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T22:45:39,557 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:45:39,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:45:39,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:39,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T22:45:39,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:45:39,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:39,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T22:45:39,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:45:39,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:39,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T22:45:39,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:45:39,589 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:39,589 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:39,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:45:39,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:39,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:39,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:45:39,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:45:39,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:39,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:45:39,599 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:39,600 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59506010, jitterRate=-0.11329135298728943}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:39,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732401939545Initializing all the Stores at 1732401939545Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401939546 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401939549 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401939549Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401939549Cleaning up temporary data from old regions at 1732401939591 (+42 ms)Region opened successfully at 1732401939601 (+10 ms) 2024-11-23T22:45:39,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:45:39,601 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:45:39,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:45:39,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:45:39,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:45:39,602 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:39,602 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732401939601Disabling compacts and flushes for region at 1732401939601Disabling writes for close at 1732401939601Writing region close event to WAL at 1732401939602 (+1 ms)Closed at 1732401939602 2024-11-23T22:45:39,603 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:39,603 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T22:45:39,603 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T22:45:39,605 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:45:39,606 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T22:45:39,624 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,37423,1732401938876] 2024-11-23T22:45:39,624 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,43635,1732401938818] 2024-11-23T22:45:39,624 DEBUG [RS:1;dd96e8dc8369:37423 {}] zookeeper.ZKUtil(111): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,624 WARN [RS:1;dd96e8dc8369:37423 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:39,625 INFO [RS:1;dd96e8dc8369:37423 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:39,625 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,625 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:45:39,629 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:45:39,629 INFO [RS:0;dd96e8dc8369:43635 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:45:39,629 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,630 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:45:39,632 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:45:39,633 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:45:39,633 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,633 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,633 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,633 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,634 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:39,635 DEBUG [RS:0;dd96e8dc8369:43635 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:39,635 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:45:39,636 INFO [RS:1;dd96e8dc8369:37423 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:45:39,636 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,640 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:45:39,640 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,640 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,641 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,641 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,641 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,641 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43635,1732401938818-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:39,641 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:45:39,641 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,641 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,641 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:39,642 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:39,643 DEBUG [RS:1;dd96e8dc8369:37423 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:39,648 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,648 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,648 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,649 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,649 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,649 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37423,1732401938876-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:39,666 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:45:39,667 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:45:39,667 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,37423,1732401938876-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,667 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,43635,1732401938818-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,667 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,667 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,667 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.Replication(171): dd96e8dc8369,43635,1732401938818 started 2024-11-23T22:45:39,667 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.Replication(171): dd96e8dc8369,37423,1732401938876 started 2024-11-23T22:45:39,690 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,690 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:39,690 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,37423,1732401938876, RpcServer on dd96e8dc8369/172.17.0.2:37423, sessionid=0x10169dfbc690002 2024-11-23T22:45:39,690 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,43635,1732401938818, RpcServer on dd96e8dc8369/172.17.0.2:43635, sessionid=0x10169dfbc690001 2024-11-23T22:45:39,690 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:45:39,690 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:45:39,690 DEBUG [RS:0;dd96e8dc8369:43635 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,690 DEBUG [RS:1;dd96e8dc8369:37423 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,690 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43635,1732401938818' 2024-11-23T22:45:39,690 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,37423,1732401938876' 2024-11-23T22:45:39,690 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:45:39,690 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:45:39,691 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:45:39,691 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:45:39,692 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:45:39,692 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:45:39,692 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:45:39,692 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:45:39,692 DEBUG [RS:1;dd96e8dc8369:37423 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:39,692 DEBUG [RS:0;dd96e8dc8369:43635 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:39,692 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,37423,1732401938876' 2024-11-23T22:45:39,692 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,43635,1732401938818' 2024-11-23T22:45:39,692 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:45:39,692 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:45:39,693 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:45:39,693 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:45:39,693 DEBUG [RS:1;dd96e8dc8369:37423 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:45:39,693 DEBUG [RS:0;dd96e8dc8369:43635 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:45:39,693 INFO [RS:1;dd96e8dc8369:37423 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:45:39,693 INFO [RS:0;dd96e8dc8369:43635 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:45:39,693 INFO [RS:1;dd96e8dc8369:37423 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:45:39,693 INFO [RS:0;dd96e8dc8369:43635 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:45:39,756 WARN [dd96e8dc8369:39879 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T22:45:39,794 INFO [RS:0;dd96e8dc8369:43635 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:39,794 INFO [RS:1;dd96e8dc8369:37423 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:39,795 INFO [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C37423%2C1732401938876, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,37423,1732401938876, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:39,795 INFO [RS:0;dd96e8dc8369:43635 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C43635%2C1732401938818, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:39,822 INFO [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,37423,1732401938876/dd96e8dc8369%2C37423%2C1732401938876.1732401939798 2024-11-23T22:45:39,822 DEBUG [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:39,822 INFO [RS:0;dd96e8dc8369:43635 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818/dd96e8dc8369%2C43635%2C1732401938818.1732401939798 2024-11-23T22:45:39,823 DEBUG [RS:0;dd96e8dc8369:43635 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:40,006 DEBUG [dd96e8dc8369:39879 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T22:45:40,007 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:45:40,007 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:45:40,008 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:45:40,008 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:45:40,008 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:45:40,008 INFO [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:45:40,008 INFO [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:45:40,008 DEBUG [dd96e8dc8369:39879 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:45:40,008 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:40,010 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,37423,1732401938876, state=OPENING 2024-11-23T22:45:40,022 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T22:45:40,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:40,033 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:40,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:40,034 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T22:45:40,034 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,37423,1732401938876}] 2024-11-23T22:45:40,035 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,189 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:45:40,194 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45369, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:45:40,198 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T22:45:40,198 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:40,199 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T22:45:40,201 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C37423%2C1732401938876.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,37423,1732401938876, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:40,219 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,37423,1732401938876/dd96e8dc8369%2C37423%2C1732401938876.meta.1732401940203.meta 2024-11-23T22:45:40,219 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:40,219 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T22:45:40,220 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T22:45:40,220 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T22:45:40,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T22:45:40,222 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T22:45:40,222 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:40,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T22:45:40,223 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T22:45:40,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:40,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T22:45:40,225 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T22:45:40,225 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:40,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T22:45:40,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T22:45:40,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T22:45:40,226 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T22:45:40,227 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:40,227 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740 2024-11-23T22:45:40,227 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T22:45:40,227 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T22:45:40,227 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T22:45:40,228 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T22:45:40,228 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64509050, jitterRate=-0.03874024748802185}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T22:45:40,229 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T22:45:40,229 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732401940220Writing region info on filesystem at 1732401940221 (+1 ms)Initializing all the Stores at 1732401940221Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401940221Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401940221Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401940221Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732401940221Cleaning up temporary data from old regions at 1732401940227 (+6 ms)Running coprocessor post-open hooks at 1732401940229 (+2 ms)Region opened successfully at 1732401940229 2024-11-23T22:45:40,230 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732401940189 2024-11-23T22:45:40,233 DEBUG [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T22:45:40,233 INFO [RS_OPEN_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T22:45:40,233 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:40,234 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as dd96e8dc8369,37423,1732401938876, state=OPEN 2024-11-23T22:45:40,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:40,242 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:40,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T22:45:40,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,243 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:40,243 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T22:45:40,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T22:45:40,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=dd96e8dc8369,37423,1732401938876 in 208 msec 2024-11-23T22:45:40,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T22:45:40,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 643 msec 2024-11-23T22:45:40,249 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T22:45:40,249 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T22:45:40,252 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:45:40,252 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,37423,1732401938876, seqNum=-1] 2024-11-23T22:45:40,253 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:40,255 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43249, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:40,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 802 msec 2024-11-23T22:45:40,262 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732401940262, completionTime=-1 2024-11-23T22:45:40,262 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T22:45:40,262 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T22:45:40,265 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T22:45:40,265 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732402000265 2024-11-23T22:45:40,265 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732402060265 2024-11-23T22:45:40,265 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-23T22:45:40,266 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T22:45:40,266 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,266 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,267 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,267 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-dd96e8dc8369:39879, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,267 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,267 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,271 DEBUG [master/dd96e8dc8369:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T22:45:40,274 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.262sec 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:40,275 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T22:45:40,276 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T22:45:40,276 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T22:45:40,277 INFO [master/dd96e8dc8369:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,39879,1732401938635-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:40,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:40,363 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request dd96e8dc8369,39879,-1 for getting cluster id 2024-11-23T22:45:40,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T22:45:40,365 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6803948-0b9e-4609-825f-8b193152163e' 2024-11-23T22:45:40,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T22:45:40,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6803948-0b9e-4609-825f-8b193152163e" 2024-11-23T22:45:40,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec4b56a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:40,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [dd96e8dc8369,39879,-1] 2024-11-23T22:45:40,366 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T22:45:40,367 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:40,368 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T22:45:40,370 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T22:45:40,370 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T22:45:40,372 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=dd96e8dc8369,37423,1732401938876, seqNum=-1] 2024-11-23T22:45:40,372 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:40,374 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:40,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:40,377 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T22:45:40,379 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:40,379 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2efa4f94 2024-11-23T22:45:40,379 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T22:45:40,381 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56810, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T22:45:40,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732401940377', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T22:45:40,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 2024-11-23T22:45:40,384 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T22:45:40,384 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732401940377" procId is: 4 2024-11-23T22:45:40,385 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T22:45:40,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:40,395 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 66e14779ed92d983105184adb4f6a295, NAME => 'testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732401940377', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1722): Closing 66e14779ed92d983105184adb4f6a295, disabling compactions & flushes 2024-11-23T22:45:40,402 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. after waiting 0 ms 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,402 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,402 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732401940377-pool-0 {}] regionserver.HRegion(1676): Region close journal for 66e14779ed92d983105184adb4f6a295: Waiting for close lock at 1732401940402Disabling compacts and flushes for region at 1732401940402Disabling writes for close at 1732401940402Writing region close event to WAL at 1732401940402Closed at 1732401940402 2024-11-23T22:45:40,403 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T22:45:40,404 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732401940404"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732401940404"}]},"ts":"1732401940404"} 2024-11-23T22:45:40,407 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T22:45:40,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T22:45:40,408 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732401940377","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732401940408"}]},"ts":"1732401940408"} 2024-11-23T22:45:40,411 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732401940377, state=ENABLING in hbase:meta 2024-11-23T22:45:40,411 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {dd96e8dc8369=0} racks are {/default-rack=0} 2024-11-23T22:45:40,411 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T22:45:40,411 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T22:45:40,412 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T22:45:40,412 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T22:45:40,412 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T22:45:40,412 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T22:45:40,412 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T22:45:40,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN}] 2024-11-23T22:45:40,413 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN 2024-11-23T22:45:40,414 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN; state=OFFLINE, location=dd96e8dc8369,43635,1732401938818; forceNewPlan=false, retain=false 2024-11-23T22:45:40,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:40,565 INFO [dd96e8dc8369:39879 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T22:45:40,566 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66e14779ed92d983105184adb4f6a295, regionState=OPENING, regionLocation=dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:40,569 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN because future has completed 2024-11-23T22:45:40,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,43635,1732401938818}] 2024-11-23T22:45:40,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:40,723 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T22:45:40,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54805, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T22:45:40,728 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,728 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e14779ed92d983105184adb4f6a295, NAME => 'testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:40,729 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732401940377 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,729 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:40,729 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,729 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,730 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,734 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e14779ed92d983105184adb4f6a295 columnFamilyName family 2024-11-23T22:45:40,734 DEBUG [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:40,735 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] regionserver.HStore(327): Store=66e14779ed92d983105184adb4f6a295/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:40,735 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,736 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,736 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,736 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,737 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,738 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,744 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T22:45:40,744 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 66e14779ed92d983105184adb4f6a295; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61722753, jitterRate=-0.080259308218956}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:45:40,745 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:40,746 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 66e14779ed92d983105184adb4f6a295: Running coprocessor pre-open hook at 1732401940729Writing region info on filesystem at 1732401940729Initializing all the Stores at 1732401940729Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401940730 (+1 ms)Cleaning up temporary data from old regions at 1732401940737 (+7 ms)Running coprocessor post-open hooks at 1732401940745 (+8 ms)Region opened successfully at 1732401940746 (+1 ms) 2024-11-23T22:45:40,746 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295., pid=6, masterSystemTime=1732401940723 2024-11-23T22:45:40,750 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=66e14779ed92d983105184adb4f6a295, regionState=OPEN, openSeqNum=2, regionLocation=dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:40,752 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,752 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:40,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,43635,1732401938818 because future has completed 2024-11-23T22:45:40,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T22:45:40,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,43635,1732401938818 in 186 msec 2024-11-23T22:45:40,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T22:45:40,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN in 346 msec 2024-11-23T22:45:40,761 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T22:45:40,761 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732401940377","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732401940761"}]},"ts":"1732401940761"} 2024-11-23T22:45:40,764 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732401940377, state=ENABLED in hbase:meta 2024-11-23T22:45:40,765 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T22:45:40,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732401940377 in 383 msec 2024-11-23T22:45:40,843 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T22:45:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39879 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T22:45:41,009 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732401940377 completed 2024-11-23T22:45:41,009 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732401940377 get assigned. Timeout = 60000ms 2024-11-23T22:45:41,009 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:45:41,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732401940377 assigned to meta. Checking AM states. 2024-11-23T22:45:41,014 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T22:45:41,014 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732401940377 assigned. 2024-11-23T22:45:41,020 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732401940377', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295., hostname=dd96e8dc8369,43635,1732401938818, seqNum=2] 2024-11-23T22:45:41,021 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T22:45:41,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T22:45:41,033 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 66e14779ed92d983105184adb4f6a295 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T22:45:41,045 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:41,065 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/.tmp/family/13c71cf37cb64acfb2e2e2ab8e1288e9 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T22:45:41,066 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/.tmp/family/13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:41,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/.tmp/family/13c71cf37cb64acfb2e2e2ab8e1288e9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:41,076 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, delay=943 ms 2024-11-23T22:45:41,077 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T22:45:41,084 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 66e14779ed92d983105184adb4f6a295 in 51ms, sequenceid=6, compaction requested=false 2024-11-23T22:45:41,084 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732401940377' 2024-11-23T22:45:41,086 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 66e14779ed92d983105184adb4f6a295: 2024-11-23T22:45:42,019 DEBUG [hfile-prefetch-1732401936427 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 13c71cf37cb64acfb2e2e2ab8e1288e9_0. 2024-11-23T22:45:42,020 DEBUG [hfile-prefetch-1732401936427 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 13c71cf37cb64acfb2e2e2ab8e1288e9_117. 2024-11-23T22:45:42,020 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, with total blocks 2, and data blocks 1 2024-11-23T22:45:42,021 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T22:45:42,021 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T22:45:42,021 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9. Will sleep 100ms and try the verification again. 2024-11-23T22:45:42,086 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;dd96e8dc8369:43635,5,FailOnTimeoutGroup] 2024-11-23T22:45:42,086 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,43635,1732401938818' ***** 2024-11-23T22:45:42,086 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-23T22:45:42,087 INFO [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T22:45:42,120 INFO [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence 2024-11-23T22:45:42,120 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T22:45:42,120 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T22:45:42,120 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T22:45:42,120 INFO [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T22:45:42,120 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T22:45:42,121 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, with total blocks 2, and data blocks 1 2024-11-23T22:45:42,121 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-23T22:45:42,121 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2293): found block for file 13c71cf37cb64acfb2e2e2ab8e1288e9 in the backing map. Acquiring read lock for offset 0 2024-11-23T22:45:42,121 DEBUG [hfile-prefetch-1732401936427 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9 has now been fully cached. 2024-11-23T22:45:42,121 DEBUG [hfile-prefetch-1732401936427 {}] hfile.PrefetchExecutor(121): Prefetch completed for 13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:42,130 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1487): Thread RS:0;dd96e8dc8369:43635 started persisting bucket cache to file 2024-11-23T22:45:42,131 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:42,131 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-23T22:45:42,139 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-23T22:45:42,139 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732401942131 2024-11-23T22:45:42,139 DEBUG [RS:0;dd96e8dc8369:43635 {}] bucket.BucketCache(1509): Thread RS:0;dd96e8dc8369:43635 finished persisting bucket cache to file, renaming 2024-11-23T22:45:42,140 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:45:42,140 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:45:42,140 INFO [RS:0;dd96e8dc8369:43635 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:45:42,140 INFO [RS:0;dd96e8dc8369:43635 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:45:42,140 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(3091): Received CLOSE for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,141 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:42,141 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:42,141 INFO [RS:0;dd96e8dc8369:43635 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;dd96e8dc8369:43635. 2024-11-23T22:45:42,141 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T22:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66e14779ed92d983105184adb4f6a295, disabling compactions & flushes 2024-11-23T22:45:42,141 DEBUG [RS:0;dd96e8dc8369:43635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:42,141 DEBUG [RS:0;dd96e8dc8369:43635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:42,141 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,141 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T22:45:42,141 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1325): Online Regions={66e14779ed92d983105184adb4f6a295=testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.} 2024-11-23T22:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. after waiting 0 ms 2024-11-23T22:45:42,141 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,141 DEBUG [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1351): Waiting on 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,146 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:42,147 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T22:45:42,148 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,148 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66e14779ed92d983105184adb4f6a295: Waiting for close lock at 1732401942141Running coprocessor pre-close hooks at 1732401942141Disabling compacts and flushes for region at 1732401942141Disabling writes for close at 1732401942141Writing region close event to WAL at 1732401942142 (+1 ms)Running coprocessor post-close hooks at 1732401942147 (+5 ms)Closed at 1732401942148 (+1 ms) 2024-11-23T22:45:42,148 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,214 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T22:45:42,214 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T22:45:42,214 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:42,214 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:42,217 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:42,217 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401942214 2024-11-23T22:45:42,218 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T22:45:42,342 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,43635,1732401938818; all regions closed. 2024-11-23T22:45:42,344 DEBUG [RS:0;dd96e8dc8369:43635 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C43635%2C1732401938818:(num 1732401939798) 2024-11-23T22:45:42,344 DEBUG [RS:0;dd96e8dc8369:43635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:45:42,344 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:45:42,344 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:42,345 INFO [RS:0;dd96e8dc8369:43635 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43635 2024-11-23T22:45:42,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:42,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,43635,1732401938818 2024-11-23T22:45:42,358 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:42,359 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,43635,1732401938818] 2024-11-23T22:45:42,379 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,43635,1732401938818 already deleted, retry=false 2024-11-23T22:45:42,379 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of dd96e8dc8369,43635,1732401938818 on dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:42,383 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure dd96e8dc8369,43635,1732401938818, splitWal=true, meta=false 2024-11-23T22:45:42,385 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure dd96e8dc8369,43635,1732401938818, splitWal=true, meta=false 2024-11-23T22:45:42,386 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for dd96e8dc8369,43635,1732401938818 (carryingMeta=false) dd96e8dc8369,43635,1732401938818/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@65f234ac[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T22:45:42,387 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): dd96e8dc8369,43635,1732401938818 had 1 regions 2024-11-23T22:45:42,387 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure dd96e8dc8369,43635,1732401938818, splitWal=true, meta=false, isMeta: false 2024-11-23T22:45:42,388 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818-splitting 2024-11-23T22:45:42,388 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818-splitting dir is empty, no logs to split. 2024-11-23T22:45:42,388 INFO [PEWorker-2 {}] master.SplitWALManager(105): dd96e8dc8369,43635,1732401938818 WAL count=0, meta=false 2024-11-23T22:45:42,390 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818-splitting dir is empty, no logs to split. 2024-11-23T22:45:42,390 INFO [PEWorker-2 {}] master.SplitWALManager(105): dd96e8dc8369,43635,1732401938818 WAL count=0, meta=false 2024-11-23T22:45:42,390 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if dd96e8dc8369,43635,1732401938818 WAL splitting is done? wals=0, meta=false 2024-11-23T22:45:42,391 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for dd96e8dc8369,43635,1732401938818 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,43635,1732401938818-splitting does not exist 2024-11-23T22:45:42,392 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN}] 2024-11-23T22:45:42,393 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN 2024-11-23T22:45:42,394 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T22:45:42,432 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T22:45:42,432 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T22:45:42,434 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T22:45:42,434 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T22:45:42,435 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T22:45:42,435 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T22:45:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:42,469 INFO [RS:0;dd96e8dc8369:43635 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:42,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43635-0x10169dfbc690001, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:42,469 INFO [RS:0;dd96e8dc8369:43635 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,43635,1732401938818; zookeeper connection closed. 2024-11-23T22:45:42,470 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4309b1c1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4309b1c1 2024-11-23T22:45:42,544 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=66e14779ed92d983105184adb4f6a295, regionState=OPENING, regionLocation=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:42,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN because future has completed 2024-11-23T22:45:42,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,37423,1732401938876}] 2024-11-23T22:45:42,608 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/dd96e8dc8369:0 server-side Connection retries=45 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T22:45:42,608 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T22:45:42,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42641 2024-11-23T22:45:42,610 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42641 connecting to ZooKeeper ensemble=127.0.0.1:51250 2024-11-23T22:45:42,611 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:42,611 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:42,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426410x0, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T22:45:42,622 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42641-0x10169dfbc690003 connected 2024-11-23T22:45:42,622 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-23T22:45:42,622 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-23T22:45:42,623 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T22:45:42,624 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache 2024-11-23T22:45:42,624 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T22:45:42,624 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T22:45:42,624 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T22:45:42,625 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T22:45:42,625 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T22:45:42,625 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T22:45:42,626 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-23T22:45:42,626 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T22:45:42,626 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-23T22:45:42,628 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T22:45:42,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42641 2024-11-23T22:45:42,629 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42641 2024-11-23T22:45:42,630 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42641 2024-11-23T22:45:42,631 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-23T22:45:42,631 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42641 2024-11-23T22:45:42,631 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T22:45:42,632 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-23T22:45:42,632 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence is complete. 2024-11-23T22:45:42,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42641 2024-11-23T22:45:42,638 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(746): ClusterId : c6803948-0b9e-4609-825f-8b193152163e 2024-11-23T22:45:42,638 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T22:45:42,644 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T22:45:42,644 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T22:45:42,654 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T22:45:42,654 DEBUG [RS:2;dd96e8dc8369:42641 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c349cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=dd96e8dc8369/172.17.0.2:0 2024-11-23T22:45:42,666 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;dd96e8dc8369:42641 2024-11-23T22:45:42,666 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T22:45:42,666 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T22:45:42,666 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T22:45:42,667 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(2659): reportForDuty to master=dd96e8dc8369,39879,1732401938635 with port=42641, startcode=1732401942607 2024-11-23T22:45:42,667 DEBUG [RS:2;dd96e8dc8369:42641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T22:45:42,669 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T22:45:42,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(363): Checking decommissioned status of RegionServer dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,670 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] master.ServerManager(517): Registering regionserver=dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,671 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9 2024-11-23T22:45:42,671 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T22:45:42,671 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T22:45:42,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:42,685 DEBUG [RS:2;dd96e8dc8369:42641 {}] zookeeper.ZKUtil(111): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,686 WARN [RS:2;dd96e8dc8369:42641 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T22:45:42,686 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [dd96e8dc8369,42641,1732401942607] 2024-11-23T22:45:42,686 INFO [RS:2;dd96e8dc8369:42641 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T22:45:42,686 DEBUG [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,692 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T22:45:42,694 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T22:45:42,694 INFO [RS:2;dd96e8dc8369:42641 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T22:45:42,694 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,694 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T22:45:42,695 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T22:45:42,695 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,695 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,695 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,695 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,695 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/dd96e8dc8369:0, corePoolSize=2, maxPoolSize=2 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/dd96e8dc8369:0, corePoolSize=1, maxPoolSize=1 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:42,696 DEBUG [RS:2;dd96e8dc8369:42641 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/dd96e8dc8369:0, corePoolSize=3, maxPoolSize=3 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,697 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42641,1732401942607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T22:45:42,702 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,703 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 66e14779ed92d983105184adb4f6a295, NAME => 'testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.', STARTKEY => '', ENDKEY => ''} 2024-11-23T22:45:42,703 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732401940377 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,703 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T22:45:42,704 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,704 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,707 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,707 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 66e14779ed92d983105184adb4f6a295 columnFamilyName family 2024-11-23T22:45:42,707 DEBUG [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T22:45:42,709 DEBUG [StoreFileOpener-66e14779ed92d983105184adb4f6a295-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9, delay=940 ms 2024-11-23T22:45:42,711 DEBUG [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:42,711 INFO [StoreOpener-66e14779ed92d983105184adb4f6a295-1 {}] regionserver.HStore(327): Store=66e14779ed92d983105184adb4f6a295/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T22:45:42,711 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,712 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,713 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,713 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,713 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,713 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T22:45:42,713 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=dd96e8dc8369,42641,1732401942607-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,713 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,713 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.Replication(171): dd96e8dc8369,42641,1732401942607 started 2024-11-23T22:45:42,714 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,715 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 66e14779ed92d983105184adb4f6a295; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70211020, jitterRate=0.04622572660446167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T22:45:42,715 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:42,716 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 66e14779ed92d983105184adb4f6a295: Running coprocessor pre-open hook at 1732401942704Writing region info on filesystem at 1732401942704Initializing all the Stores at 1732401942704Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732401942704Cleaning up temporary data from old regions at 1732401942713 (+9 ms)Running coprocessor post-open hooks at 1732401942715 (+2 ms)Region opened successfully at 1732401942715 2024-11-23T22:45:42,716 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295., pid=9, masterSystemTime=1732401942700 2024-11-23T22:45:42,719 DEBUG [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,719 INFO [RS_OPEN_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:42,719 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=66e14779ed92d983105184adb4f6a295, regionState=OPEN, openSeqNum=10, regionLocation=dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:42,722 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39879 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=dd96e8dc8369,37423,1732401938876, table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-23T22:45:42,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,37423,1732401938876 because future has completed 2024-11-23T22:45:42,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T22:45:42,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 66e14779ed92d983105184adb4f6a295, server=dd96e8dc8369,37423,1732401938876 in 176 msec 2024-11-23T22:45:42,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T22:45:42,729 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server dd96e8dc8369,43635,1732401938818 after splitting done 2024-11-23T22:45:42,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732401940377, region=66e14779ed92d983105184adb4f6a295, ASSIGN in 334 msec 2024-11-23T22:45:42,730 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T22:45:42,730 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1482): Serving as dd96e8dc8369,42641,1732401942607, RpcServer on dd96e8dc8369/172.17.0.2:42641, sessionid=0x10169dfbc690003 2024-11-23T22:45:42,730 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T22:45:42,730 DEBUG [RS:2;dd96e8dc8369:42641 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,730 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,42641,1732401942607' 2024-11-23T22:45:42,731 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T22:45:42,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure dd96e8dc8369,43635,1732401938818, splitWal=true, meta=false in 349 msec 2024-11-23T22:45:42,731 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T22:45:42,732 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T22:45:42,732 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T22:45:42,732 DEBUG [RS:2;dd96e8dc8369:42641 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:42,732 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'dd96e8dc8369,42641,1732401942607' 2024-11-23T22:45:42,732 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T22:45:42,733 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T22:45:42,733 DEBUG [RS:2;dd96e8dc8369:42641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T22:45:42,733 INFO [RS:2;dd96e8dc8369:42641 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T22:45:42,733 INFO [RS:2;dd96e8dc8369:42641 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T22:45:42,834 INFO [RS:2;dd96e8dc8369:42641 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T22:45:42,834 INFO [RS:2;dd96e8dc8369:42641 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=dd96e8dc8369%2C42641%2C1732401942607, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,42641,1732401942607, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs, maxLogs=55 2024-11-23T22:45:42,850 INFO [RS:2;dd96e8dc8369:42641 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/WALs/dd96e8dc8369,42641,1732401942607/dd96e8dc8369%2C42641%2C1732401942607.1732401942835 2024-11-23T22:45:42,850 DEBUG [RS:2;dd96e8dc8369:42641 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T22:45:43,231 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T22:45:43,231 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:45:43,231 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:43,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T22:45:43,232 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T22:45:43,232 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=970552506, stopped=false 2024-11-23T22:45:43,232 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=dd96e8dc8369,39879,1732401938635 2024-11-23T22:45:43,303 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:43,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:43,303 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:43,303 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:45:43,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T22:45:43,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:43,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:43,304 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T22:45:43,304 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:43,305 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,305 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,37423,1732401938876' ***** 2024-11-23T22:45:43,305 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:45:43,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:43,305 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'dd96e8dc8369,42641,1732401942607' ***** 2024-11-23T22:45:43,305 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T22:45:43,305 INFO [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T22:45:43,306 INFO [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T22:45:43,306 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T22:45:43,306 INFO [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence 2024-11-23T22:45:43,306 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T22:45:43,306 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:43,306 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T22:45:43,306 INFO [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T22:45:43,306 INFO [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2e1fb120-8c49-90a0-0909-5881a90c3e08/bucket.persistence 2024-11-23T22:45:43,306 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T22:45:43,306 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T22:45:43,307 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T22:45:43,307 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T22:45:43,307 INFO [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T22:45:43,307 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T22:45:43,307 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T22:45:43,316 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1487): Thread RS:2;dd96e8dc8369:42641 started persisting bucket cache to file 2024-11-23T22:45:43,317 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:43,317 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:43,317 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1487): Thread RS:1;dd96e8dc8369:37423 started persisting bucket cache to file 2024-11-23T22:45:43,317 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T22:45:43,317 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T22:45:43,320 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:43,320 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T22:45:43,320 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401943317 2024-11-23T22:45:43,320 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732401943317 2024-11-23T22:45:43,320 DEBUG [RS:2;dd96e8dc8369:42641 {}] bucket.BucketCache(1509): Thread RS:2;dd96e8dc8369:42641 finished persisting bucket cache to file, renaming 2024-11-23T22:45:43,320 DEBUG [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1509): Thread RS:1;dd96e8dc8369:37423 finished persisting bucket cache to file, renaming 2024-11-23T22:45:43,320 WARN [RS:1;dd96e8dc8369:37423 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:45:43,320 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:45:43,320 INFO [RS:1;dd96e8dc8369:37423 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T22:45:43,320 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:45:43,320 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T22:45:43,320 INFO [RS:1;dd96e8dc8369:37423 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:43,320 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(3091): Received CLOSE for 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:43,320 INFO [RS:2;dd96e8dc8369:42641 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;dd96e8dc8369:42641. 2024-11-23T22:45:43,321 DEBUG [RS:2;dd96e8dc8369:42641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(959): stopping server dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:43,321 DEBUG [RS:2;dd96e8dc8369:42641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;dd96e8dc8369:37423. 2024-11-23T22:45:43,321 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,42641,1732401942607; all regions closed. 2024-11-23T22:45:43,321 DEBUG [RS:1;dd96e8dc8369:37423 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T22:45:43,321 DEBUG [RS:1;dd96e8dc8369:37423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,321 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:45:43,321 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 66e14779ed92d983105184adb4f6a295, disabling compactions & flushes 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:45:43,321 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:45:43,321 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T22:45:43,321 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. after waiting 0 ms 2024-11-23T22:45:43,321 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:43,321 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T22:45:43,321 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 66e14779ed92d983105184adb4f6a295=testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295.} 2024-11-23T22:45:43,322 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T22:45:43,322 DEBUG [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 66e14779ed92d983105184adb4f6a295 2024-11-23T22:45:43,322 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T22:45:43,322 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T22:45:43,322 DEBUG [StoreFileCloser-66e14779ed92d983105184adb4f6a295-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/family/13c71cf37cb64acfb2e2e2ab8e1288e9 2024-11-23T22:45:43,322 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T22:45:43,322 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T22:45:43,322 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T22:45:43,322 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-23T22:45:43,322 DEBUG [RS:2;dd96e8dc8369:42641 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:43,322 INFO [RS:2;dd96e8dc8369:42641 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C42641%2C1732401942607:(num 1732401942835) 2024-11-23T22:45:43,323 DEBUG [RS:2;dd96e8dc8369:42641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T22:45:43,323 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T22:45:43,323 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:43,324 INFO [RS:2;dd96e8dc8369:42641 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42641 2024-11-23T22:45:43,326 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/default/testBlockEvictionOnGracefulStop1732401940377/66e14779ed92d983105184adb4f6a295/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T22:45:43,326 INFO [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:43,327 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 66e14779ed92d983105184adb4f6a295: Waiting for close lock at 1732401943321Running coprocessor pre-close hooks at 1732401943321Disabling compacts and flushes for region at 1732401943321Disabling writes for close at 1732401943321Writing region close event to WAL at 1732401943322 (+1 ms)Running coprocessor post-close hooks at 1732401943326 (+4 ms)Closed at 1732401943326 2024-11-23T22:45:43,327 DEBUG [RS_CLOSE_REGION-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295. 2024-11-23T22:45:43,331 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9fb041aca185405db33dc1f3f547bd40 2024-11-23T22:45:43,347 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/9fb041aca185405db33dc1f3f547bd40 is 209, key is testBlockEvictionOnGracefulStop1732401940377,,1732401940381.66e14779ed92d983105184adb4f6a295./info:regioninfo/1732401942719/Put/seqid=0 2024-11-23T22:45:43,347 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/9fb041aca185405db33dc1f3f547bd40 2024-11-23T22:45:43,352 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:43,355 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 874db37aef9a4c2390e13fe8fa6dbb95 2024-11-23T22:45:43,367 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/874db37aef9a4c2390e13fe8fa6dbb95 is 43, key is default/ns:d/1732401940255/Put/seqid=0 2024-11-23T22:45:43,368 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/874db37aef9a4c2390e13fe8fa6dbb95 2024-11-23T22:45:43,376 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2d9231c9f47b464ca7285643b02b3039 2024-11-23T22:45:43,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,42641,1732401942607 2024-11-23T22:45:43,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:43,388 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/2d9231c9f47b464ca7285643b02b3039 is 80, key is testBlockEvictionOnGracefulStop1732401940377/table:state/1732401940761/Put/seqid=0 2024-11-23T22:45:43,388 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:43,389 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/2d9231c9f47b464ca7285643b02b3039 2024-11-23T22:45:43,389 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,42641,1732401942607] 2024-11-23T22:45:43,391 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/info/9fb041aca185405db33dc1f3f547bd40 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/9fb041aca185405db33dc1f3f547bd40 2024-11-23T22:45:43,392 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/9fb041aca185405db33dc1f3f547bd40, delay=913 ms 2024-11-23T22:45:43,393 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/9fb041aca185405db33dc1f3f547bd40, entries=14, sequenceid=13, filesize=7.5 K 2024-11-23T22:45:43,393 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/ns/874db37aef9a4c2390e13fe8fa6dbb95 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/874db37aef9a4c2390e13fe8fa6dbb95 2024-11-23T22:45:43,394 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/874db37aef9a4c2390e13fe8fa6dbb95, delay=984 ms 2024-11-23T22:45:43,395 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/874db37aef9a4c2390e13fe8fa6dbb95, entries=2, sequenceid=13, filesize=5.0 K 2024-11-23T22:45:43,396 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/.tmp/table/2d9231c9f47b464ca7285643b02b3039 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/2d9231c9f47b464ca7285643b02b3039 2024-11-23T22:45:43,397 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/2d9231c9f47b464ca7285643b02b3039, delay=982 ms 2024-11-23T22:45:43,398 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/2d9231c9f47b464ca7285643b02b3039, entries=2, sequenceid=13, filesize=5.3 K 2024-11-23T22:45:43,398 INFO [regionserver/dd96e8dc8369:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:43,398 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=13, compaction requested=false 2024-11-23T22:45:43,399 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/info/9fb041aca185405db33dc1f3f547bd40 2024-11-23T22:45:43,400 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/ns/874db37aef9a4c2390e13fe8fa6dbb95 2024-11-23T22:45:43,400 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/table/2d9231c9f47b464ca7285643b02b3039 2024-11-23T22:45:43,404 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-23T22:45:43,404 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T22:45:43,404 INFO [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:43,405 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732401943321Running coprocessor pre-close hooks at 1732401943322 (+1 ms)Disabling compacts and flushes for region at 1732401943322Disabling writes for close at 1732401943322Obtaining lock to block concurrent updates at 1732401943322Preparing flush snapshotting stores in 1588230740 at 1732401943322Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732401943322Flushing stores of hbase:meta,,1.1588230740 at 1732401943323 (+1 ms)Flushing 1588230740/info: creating writer at 1732401943323Flushing 1588230740/info: appending metadata at 1732401943346 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732401943346Flushing 1588230740/ns: creating writer at 1732401943350 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732401943367 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732401943367Flushing 1588230740/table: creating writer at 1732401943370 (+3 ms)Flushing 1588230740/table: appending metadata at 1732401943387 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732401943387Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aaa6603: reopening flushed file at 1732401943391 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@408809ea: reopening flushed file at 1732401943393 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2672b0a5: reopening flushed file at 1732401943395 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 76ms, sequenceid=13, compaction requested=false at 1732401943398 (+3 ms)Writing region close event to WAL at 1732401943400 (+2 ms)Running coprocessor post-close hooks at 1732401943404 (+4 ms)Closed at 1732401943404 2024-11-23T22:45:43,405 DEBUG [RS_CLOSE_META-regionserver/dd96e8dc8369:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T22:45:43,421 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,42641,1732401942607 already deleted, retry=false 2024-11-23T22:45:43,421 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,42641,1732401942607 expired; onlineServers=1 2024-11-23T22:45:43,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,511 INFO [RS:2;dd96e8dc8369:42641 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:43,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42641-0x10169dfbc690003, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,511 INFO [RS:2;dd96e8dc8369:42641 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,42641,1732401942607; zookeeper connection closed. 2024-11-23T22:45:43,512 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@699a6c8c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@699a6c8c 2024-11-23T22:45:43,522 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(976): stopping server dd96e8dc8369,37423,1732401938876; all regions closed. 2024-11-23T22:45:43,526 DEBUG [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:43,527 INFO [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C37423%2C1732401938876.meta:.meta(num 1732401940203) 2024-11-23T22:45:43,531 DEBUG [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/oldWALs 2024-11-23T22:45:43,531 INFO [RS:1;dd96e8dc8369:37423 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL dd96e8dc8369%2C37423%2C1732401938876:(num 1732401939798) 2024-11-23T22:45:43,531 DEBUG [RS:1;dd96e8dc8369:37423 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T22:45:43,531 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T22:45:43,531 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:43,532 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.ChoreService(370): Chore service for: regionserver/dd96e8dc8369:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:43,532 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:43,532 INFO [regionserver/dd96e8dc8369:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:43,532 INFO [RS:1;dd96e8dc8369:37423 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37423 2024-11-23T22:45:43,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T22:45:43,545 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/dd96e8dc8369,37423,1732401938876 2024-11-23T22:45:43,545 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:43,546 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [dd96e8dc8369,37423,1732401938876] 2024-11-23T22:45:43,569 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/dd96e8dc8369,37423,1732401938876 already deleted, retry=false 2024-11-23T22:45:43,569 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; dd96e8dc8369,37423,1732401938876 expired; onlineServers=0 2024-11-23T22:45:43,569 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'dd96e8dc8369,39879,1732401938635' ***** 2024-11-23T22:45:43,569 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T22:45:43,569 INFO [M:0;dd96e8dc8369:39879 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T22:45:43,569 INFO [M:0;dd96e8dc8369:39879 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T22:45:43,569 DEBUG [M:0;dd96e8dc8369:39879 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T22:45:43,570 DEBUG [M:0;dd96e8dc8369:39879 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T22:45:43,570 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T22:45:43,570 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401939483 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.large.0-1732401939483,5,FailOnTimeoutGroup] 2024-11-23T22:45:43,570 DEBUG [master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401939483 {}] cleaner.HFileCleaner(306): Exit Thread[master/dd96e8dc8369:0:becomeActiveMaster-HFileCleaner.small.0-1732401939483,5,FailOnTimeoutGroup] 2024-11-23T22:45:43,570 INFO [M:0;dd96e8dc8369:39879 {}] hbase.ChoreService(370): Chore service for: master/dd96e8dc8369:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T22:45:43,570 INFO [M:0;dd96e8dc8369:39879 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T22:45:43,571 DEBUG [M:0;dd96e8dc8369:39879 {}] master.HMaster(1795): Stopping service threads 2024-11-23T22:45:43,571 INFO [M:0;dd96e8dc8369:39879 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T22:45:43,571 INFO [M:0;dd96e8dc8369:39879 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T22:45:43,571 INFO [M:0;dd96e8dc8369:39879 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T22:45:43,572 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T22:45:43,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T22:45:43,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T22:45:43,580 DEBUG [M:0;dd96e8dc8369:39879 {}] zookeeper.ZKUtil(347): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T22:45:43,580 WARN [M:0;dd96e8dc8369:39879 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T22:45:43,581 INFO [M:0;dd96e8dc8369:39879 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/.lastflushedseqids 2024-11-23T22:45:43,586 INFO [M:0;dd96e8dc8369:39879 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T22:45:43,586 INFO [M:0;dd96e8dc8369:39879 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T22:45:43,586 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T22:45:43,586 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:43,586 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:43,586 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T22:45:43,586 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:43,586 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.81 KB heapSize=41.05 KB 2024-11-23T22:45:43,595 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9569d7a4cf5a40d8bd730470dfcfcb45 2024-11-23T22:45:43,607 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9569d7a4cf5a40d8bd730470dfcfcb45 is 82, key is hbase:meta,,1/info:regioninfo/1732401940233/Put/seqid=0 2024-11-23T22:45:43,608 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9569d7a4cf5a40d8bd730470dfcfcb45 2024-11-23T22:45:43,614 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5759794558d04f76adfb97b9a4ed2174 2024-11-23T22:45:43,630 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5759794558d04f76adfb97b9a4ed2174 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732401940766/Put/seqid=0 2024-11-23T22:45:43,631 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.08 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5759794558d04f76adfb97b9a4ed2174 2024-11-23T22:45:43,638 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9770a237803f4c8cbfaa9b1ef774fa3d 2024-11-23T22:45:43,650 DEBUG [M:0;dd96e8dc8369:39879 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9770a237803f4c8cbfaa9b1ef774fa3d is 69, key is dd96e8dc8369,37423,1732401938876/rs:state/1732401939554/Put/seqid=0 2024-11-23T22:45:43,651 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9770a237803f4c8cbfaa9b1ef774fa3d 2024-11-23T22:45:43,652 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9770a237803f4c8cbfaa9b1ef774fa3d 2024-11-23T22:45:43,653 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9569d7a4cf5a40d8bd730470dfcfcb45 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9569d7a4cf5a40d8bd730470dfcfcb45 2024-11-23T22:45:43,654 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9569d7a4cf5a40d8bd730470dfcfcb45, entries=8, sequenceid=83, filesize=5.5 K 2024-11-23T22:45:43,655 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5759794558d04f76adfb97b9a4ed2174 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5759794558d04f76adfb97b9a4ed2174 2024-11-23T22:45:43,656 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5759794558d04f76adfb97b9a4ed2174, entries=9, sequenceid=83, filesize=6.4 K 2024-11-23T22:45:43,656 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9770a237803f4c8cbfaa9b1ef774fa3d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9770a237803f4c8cbfaa9b1ef774fa3d 2024-11-23T22:45:43,658 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9770a237803f4c8cbfaa9b1ef774fa3d 2024-11-23T22:45:43,658 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/af193b82-7dc1-bd76-0589-e9cf25a78b35/b4cde35e-0cf5-5a1f-b644-bccae6be91e9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9770a237803f4c8cbfaa9b1ef774fa3d, entries=3, sequenceid=83, filesize=5.3 K 2024-11-23T22:45:43,658 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=83, compaction requested=false 2024-11-23T22:45:43,658 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,658 INFO [RS:1;dd96e8dc8369:37423 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:43,658 DEBUG [pool-170-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37423-0x10169dfbc690002, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,659 INFO [RS:1;dd96e8dc8369:37423 {}] regionserver.HRegionServer(1031): Exiting; stopping=dd96e8dc8369,37423,1732401938876; zookeeper connection closed. 2024-11-23T22:45:43,659 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@c32199c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@c32199c 2024-11-23T22:45:43,659 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T22:45:43,660 INFO [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T22:45:43,660 DEBUG [M:0;dd96e8dc8369:39879 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732401943586Disabling compacts and flushes for region at 1732401943586Disabling writes for close at 1732401943586Obtaining lock to block concurrent updates at 1732401943586Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732401943586Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33602, getHeapSize=41976, getOffHeapSize=0, getCellsCount=98 at 1732401943587 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732401943587Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732401943587Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732401943607 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732401943607Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732401943609 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732401943630 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732401943630Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732401943632 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732401943650 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732401943650Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c93088: reopening flushed file at 1732401943653 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14675228: reopening flushed file at 1732401943654 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f6cae07: reopening flushed file at 1732401943656 (+2 ms)Finished flush of dataSize ~32.81 KB/33602, heapSize ~40.76 KB/41736, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 72ms, sequenceid=83, compaction requested=false at 1732401943658 (+2 ms)Writing region close event to WAL at 1732401943660 (+2 ms)Closed at 1732401943660 2024-11-23T22:45:43,661 INFO [M:0;dd96e8dc8369:39879 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T22:45:43,661 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T22:45:43,661 INFO [M:0;dd96e8dc8369:39879 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39879 2024-11-23T22:45:43,661 INFO [M:0;dd96e8dc8369:39879 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T22:45:43,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,774 INFO [M:0;dd96e8dc8369:39879 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T22:45:43,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39879-0x10169dfbc690000, quorum=127.0.0.1:51250, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T22:45:43,779 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T22:45:43,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T22:45:43,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T22:45:43,790 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=444 (was 457), ProcessCount=11 (was 11), AvailableMemoryMB=8476 (was 7213) - AvailableMemoryMB LEAK? -