2024-11-16 09:48:19,985 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 09:48:20,003 main DEBUG Took 0.010061 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 09:48:20,003 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 09:48:20,004 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 09:48:20,005 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 09:48:20,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,014 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 09:48:20,026 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,029 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,030 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,031 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,032 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,034 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,035 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,035 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,036 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,037 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,037 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,038 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,038 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 09:48:20,039 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,039 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 09:48:20,040 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 09:48:20,042 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 09:48:20,044 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 09:48:20,045 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 09:48:20,046 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 09:48:20,046 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 09:48:20,055 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 09:48:20,057 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 09:48:20,059 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 09:48:20,059 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 09:48:20,059 main DEBUG createAppenders(={Console}) 2024-11-16 09:48:20,060 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 09:48:20,060 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 09:48:20,060 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 09:48:20,061 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 09:48:20,061 main DEBUG OutputStream closed 2024-11-16 09:48:20,062 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 09:48:20,062 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 09:48:20,062 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 09:48:20,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 09:48:20,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 09:48:20,146 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 09:48:20,147 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 09:48:20,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 09:48:20,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 09:48:20,148 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 09:48:20,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 09:48:20,149 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 09:48:20,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 09:48:20,150 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 09:48:20,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 09:48:20,151 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 09:48:20,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 09:48:20,152 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 09:48:20,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 09:48:20,153 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 09:48:20,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 09:48:20,158 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 09:48:20,158 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 09:48:20,159 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 09:48:20,160 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T09:48:20,179 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-16 09:48:20,183 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 09:48:20,184 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T09:48:20,460 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d 2024-11-16T09:48:20,485 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=623, ProcessCount=11, AvailableMemoryMB=7099 2024-11-16T09:48:20,585 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T09:48:20,682 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:20,697 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/cluster_16cd28aa-b4ea-31d6-a049-e45c54456c57, deleteOnExit=true 2024-11-16T09:48:20,798 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/cluster_16cd28aa-b4ea-31d6-a049-e45c54456c57/zookeeper_0, clientPort=59347, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/cluster_16cd28aa-b4ea-31d6-a049-e45c54456c57/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/cluster_16cd28aa-b4ea-31d6-a049-e45c54456c57/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T09:48:20,812 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59347 2024-11-16T09:48:20,813 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:20,912 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 with version=8 2024-11-16T09:48:20,913 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase-staging 2024-11-16T09:48:21,017 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T09:48:21,301 INFO [Time-limited test {}] client.ConnectionUtils(128): master/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:21,312 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,314 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,324 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:21,324 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,325 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:21,474 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T09:48:21,561 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:48:21,598 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T09:48:21,608 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T09:48:21,612 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:21,652 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20984 (auto-detected) 2024-11-16T09:48:21,653 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T09:48:21,679 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43181 2024-11-16T09:48:21,704 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43181 connecting to ZooKeeper ensemble=127.0.0.1:59347 2024-11-16T09:48:21,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:431810x0, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:21,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43181-0x101b07b0f070000 connected 2024-11-16T09:48:21,769 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:21,770 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:21,778 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:21,782 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33, hbase.cluster.distributed=false 2024-11-16T09:48:21,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:21,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43181 2024-11-16T09:48:21,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43181 2024-11-16T09:48:21,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43181 2024-11-16T09:48:21,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43181 2024-11-16T09:48:21,821 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43181 2024-11-16T09:48:21,909 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:21,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,911 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:21,911 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,912 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:21,915 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:48:21,917 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:21,918 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42857 2024-11-16T09:48:21,920 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42857 connecting to ZooKeeper ensemble=127.0.0.1:59347 2024-11-16T09:48:21,920 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:21,921 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:21,925 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428570x0, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:21,926 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42857-0x101b07b0f070001 connected 2024-11-16T09:48:21,926 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:21,932 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:48:21,944 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.cache 2024-11-16T09:48:21,949 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T09:48:21,951 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T09:48:21,952 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T09:48:21,954 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T09:48:21,954 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T09:48:21,959 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T09:48:21,964 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:48:21,966 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T09:48:21,967 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence is complete. 2024-11-16T09:48:21,967 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T09:48:21,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:21,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42857 2024-11-16T09:48:21,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42857 2024-11-16T09:48:21,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42857 2024-11-16T09:48:21,977 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42857 2024-11-16T09:48:21,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42857 2024-11-16T09:48:21,997 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:21,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,997 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,998 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:21,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:21,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:21,998 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:48:21,998 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:21,999 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44683 2024-11-16T09:48:22,001 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44683 connecting to ZooKeeper ensemble=127.0.0.1:59347 2024-11-16T09:48:22,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:22,002 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:22,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446830x0, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:22,008 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446830x0, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:22,008 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44683-0x101b07b0f070002 connected 2024-11-16T09:48:22,009 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:48:22,009 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.cache 2024-11-16T09:48:22,010 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T09:48:22,010 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T09:48:22,010 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T09:48:22,011 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T09:48:22,011 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T09:48:22,011 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T09:48:22,011 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T09:48:22,011 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence is complete. 2024-11-16T09:48:22,012 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:48:22,013 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T09:48:22,015 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:22,020 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44683 2024-11-16T09:48:22,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44683 2024-11-16T09:48:22,023 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44683 2024-11-16T09:48:22,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44683 2024-11-16T09:48:22,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44683 2024-11-16T09:48:22,042 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;037c58a19ddb:43181 2024-11-16T09:48:22,043 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,055 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:22,089 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:22,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,091 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T09:48:22,093 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/037c58a19ddb,43181,1731750501093 from backup master directory 2024-11-16T09:48:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,096 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:22,097 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:22,097 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,101 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T09:48:22,103 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T09:48:22,113 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase.id] with ID: 650bf649-60a7-42f3-86e3-14fb1f857542 2024-11-16T09:48:22,114 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.tmp/hbase.id 2024-11-16T09:48:22,122 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase.id] 2024-11-16T09:48:22,164 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:22,169 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T09:48:22,190 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-16T09:48:22,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,222 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:48:22,227 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T09:48:22,279 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:48:22,285 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:22,346 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store 2024-11-16T09:48:22,367 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T09:48:22,371 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:22,372 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T09:48:22,372 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:22,373 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:22,375 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T09:48:22,375 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:22,375 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:22,376 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731750502372Disabling compacts and flushes for region at 1731750502372Disabling writes for close at 1731750502375 (+3 ms)Writing region close event to WAL at 1731750502375Closed at 1731750502375 2024-11-16T09:48:22,376 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/.initializing 2024-11-16T09:48:22,377 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,398 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:22,412 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C43181%2C1731750501093, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,43181,1731750501093, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/oldWALs, maxLogs=10 2024-11-16T09:48:22,461 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,43181,1731750501093/037c58a19ddb%2C43181%2C1731750501093.1731750502417 2024-11-16T09:48:22,462 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:22,462 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:22,463 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:22,466 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,468 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,511 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,542 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T09:48:22,546 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:22,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:22,549 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,554 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T09:48:22,554 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:22,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:22,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,559 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T09:48:22,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:22,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:22,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T09:48:22,565 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:22,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:22,566 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,569 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,570 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,573 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,574 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,576 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:22,579 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:22,586 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:22,587 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66199184, jitterRate=-0.013555288314819336}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:22,593 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731750502481Initializing all the Stores at 1731750502482 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750502482Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750502484 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750502484Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750502484Cleaning up temporary data from old regions at 1731750502574 (+90 ms)Region opened successfully at 1731750502593 (+19 ms) 2024-11-16T09:48:22,594 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T09:48:22,628 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ebde83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:22,665 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T09:48:22,683 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T09:48:22,683 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T09:48:22,687 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T09:48:22,688 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T09:48:22,696 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-16T09:48:22,697 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T09:48:22,728 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T09:48:22,740 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T09:48:22,742 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T09:48:22,746 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T09:48:22,748 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T09:48:22,750 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T09:48:22,752 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T09:48:22,755 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T09:48:22,757 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T09:48:22,759 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T09:48:22,760 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T09:48:22,779 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T09:48:22,781 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T09:48:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:22,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,799 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=037c58a19ddb,43181,1731750501093, sessionid=0x101b07b0f070000, setting cluster-up flag (Was=false) 2024-11-16T09:48:22,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,822 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T09:48:22,824 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,828 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:22,836 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T09:48:22,837 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,43181,1731750501093 2024-11-16T09:48:22,843 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T09:48:22,924 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:22,931 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(746): ClusterId : 650bf649-60a7-42f3-86e3-14fb1f857542 2024-11-16T09:48:22,931 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(746): ClusterId : 650bf649-60a7-42f3-86e3-14fb1f857542 2024-11-16T09:48:22,935 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:48:22,935 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:48:22,937 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T09:48:22,941 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:48:22,941 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:48:22,941 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:48:22,941 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:48:22,945 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:48:22,946 DEBUG [RS:0;037c58a19ddb:42857 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3eeb6d78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:22,947 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T09:48:22,948 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:48:22,949 DEBUG [RS:1;037c58a19ddb:44683 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5db3c364, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:22,955 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 037c58a19ddb,43181,1731750501093 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T09:48:22,964 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:22,964 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:22,965 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:22,965 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:22,965 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/037c58a19ddb:0, corePoolSize=10, maxPoolSize=10 2024-11-16T09:48:22,965 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:22,965 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:22,966 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:22,968 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;037c58a19ddb:44683 2024-11-16T09:48:22,971 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;037c58a19ddb:42857 2024-11-16T09:48:22,973 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:48:22,973 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:48:22,973 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750532972 2024-11-16T09:48:22,973 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:48:22,973 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:48:22,973 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:48:22,973 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:48:22,973 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:22,974 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T09:48:22,975 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T09:48:22,976 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T09:48:22,978 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,43181,1731750501093 with port=44683, startcode=1731750501996 2024-11-16T09:48:22,980 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,43181,1731750501093 with port=42857, startcode=1731750501869 2024-11-16T09:48:22,980 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T09:48:22,980 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:22,981 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T09:48:22,981 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T09:48:22,981 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T09:48:22,981 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T09:48:22,984 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:22,988 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T09:48:22,990 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T09:48:22,991 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T09:48:22,994 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T09:48:22,995 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T09:48:22,997 DEBUG [RS:1;037c58a19ddb:44683 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:48:22,997 DEBUG [RS:0;037c58a19ddb:42857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:48:23,000 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750502996,5,FailOnTimeoutGroup] 2024-11-16T09:48:23,002 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750503000,5,FailOnTimeoutGroup] 2024-11-16T09:48:23,002 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,002 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T09:48:23,003 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,004 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,011 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T09:48:23,012 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:23,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:23,025 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T09:48:23,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:48:23,031 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T09:48:23,037 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:48:23,038 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,038 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T09:48:23,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:48:23,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T09:48:23,058 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:48:23,058 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,059 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:48:23,060 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:23,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:23,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:48:23,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:48:23,062 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:23,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:48:23,080 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:23,081 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62579795, jitterRate=-0.06748838722705841}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:23,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731750503022Initializing all the Stores at 1731750503022Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503022Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503025 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750503025Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503025Cleaning up temporary data from old regions at 1731750503061 (+36 ms)Region opened successfully at 1731750503085 (+24 ms) 2024-11-16T09:48:23,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:48:23,086 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33199, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:48:23,086 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:48:23,086 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52309, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:48:23,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:48:23,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T09:48:23,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:48:23,089 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:23,089 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731750503085Disabling compacts and flushes for region at 1731750503085Disabling writes for close at 1731750503086 (+1 ms)Writing region close event to WAL at 1731750503088 (+2 ms)Closed at 1731750503088 2024-11-16T09:48:23,092 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:23,092 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43181 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,092 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T09:48:23,095 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43181 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T09:48:23,108 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43181 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,108 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43181 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,112 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:23,113 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T09:48:23,113 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:48:23,113 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:23,116 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:23,116 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T09:48:23,116 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:48:23,117 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T09:48:23,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:23,123 DEBUG [RS:1;037c58a19ddb:44683 {}] zookeeper.ZKUtil(111): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,123 WARN [RS:1;037c58a19ddb:44683 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:23,123 INFO [RS:1;037c58a19ddb:44683 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:23,123 DEBUG [RS:0;037c58a19ddb:42857 {}] zookeeper.ZKUtil(111): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,124 WARN [RS:0;037c58a19ddb:42857 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:23,124 INFO [RS:0;037c58a19ddb:42857 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:23,124 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,125 DEBUG [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,125 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,42857,1731750501869] 2024-11-16T09:48:23,125 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,44683,1731750501996] 2024-11-16T09:48:23,161 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:48:23,161 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:48:23,180 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:48:23,180 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:48:23,186 INFO [RS:0;037c58a19ddb:42857 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:48:23,186 INFO [RS:1;037c58a19ddb:44683 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:48:23,187 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,186 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,188 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:48:23,188 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:48:23,194 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:48:23,194 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:48:23,196 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,196 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,196 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,196 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,196 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,196 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,196 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,197 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:23,197 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:23,197 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,198 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,199 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,199 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:23,199 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:23,199 DEBUG [RS:1;037c58a19ddb:44683 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:23,199 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:23,199 DEBUG [RS:0;037c58a19ddb:42857 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:23,204 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,204 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,204 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,204 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,205 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,205 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,44683,1731750501996-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:23,206 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,206 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,206 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,206 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,207 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,207 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,42857,1731750501869-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:23,239 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:48:23,239 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:48:23,242 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,42857,1731750501869-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,242 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,44683,1731750501996-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,242 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,242 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,243 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.Replication(171): 037c58a19ddb,42857,1731750501869 started 2024-11-16T09:48:23,243 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.Replication(171): 037c58a19ddb,44683,1731750501996 started 2024-11-16T09:48:23,269 WARN [037c58a19ddb:43181 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T09:48:23,270 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,270 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,271 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,42857,1731750501869, RpcServer on 037c58a19ddb/172.17.0.2:42857, sessionid=0x101b07b0f070001 2024-11-16T09:48:23,271 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,44683,1731750501996, RpcServer on 037c58a19ddb/172.17.0.2:44683, sessionid=0x101b07b0f070002 2024-11-16T09:48:23,272 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:48:23,272 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:48:23,272 DEBUG [RS:1;037c58a19ddb:44683 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,272 DEBUG [RS:0;037c58a19ddb:42857 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,272 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,44683,1731750501996' 2024-11-16T09:48:23,272 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,42857,1731750501869' 2024-11-16T09:48:23,272 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:48:23,273 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:48:23,274 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:48:23,274 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:48:23,274 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:48:23,275 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:48:23,275 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:48:23,275 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:48:23,275 DEBUG [RS:0;037c58a19ddb:42857 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,275 DEBUG [RS:1;037c58a19ddb:44683 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,44683,1731750501996 2024-11-16T09:48:23,275 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,42857,1731750501869' 2024-11-16T09:48:23,275 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,44683,1731750501996' 2024-11-16T09:48:23,275 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:48:23,275 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:48:23,276 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:48:23,276 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:48:23,277 DEBUG [RS:0;037c58a19ddb:42857 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:48:23,277 DEBUG [RS:1;037c58a19ddb:44683 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:48:23,277 INFO [RS:1;037c58a19ddb:44683 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:48:23,277 INFO [RS:0;037c58a19ddb:42857 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:48:23,277 INFO [RS:1;037c58a19ddb:44683 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:48:23,277 INFO [RS:0;037c58a19ddb:42857 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:48:23,384 INFO [RS:1;037c58a19ddb:44683 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:23,384 INFO [RS:0;037c58a19ddb:42857 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:23,385 INFO [RS:1;037c58a19ddb:44683 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C44683%2C1731750501996, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,44683,1731750501996, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:23,385 INFO [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C42857%2C1731750501869, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,42857,1731750501869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:23,407 INFO [RS:1;037c58a19ddb:44683 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,44683,1731750501996/037c58a19ddb%2C44683%2C1731750501996.1731750503388 2024-11-16T09:48:23,408 DEBUG [RS:1;037c58a19ddb:44683 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:23,416 INFO [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,42857,1731750501869/037c58a19ddb%2C42857%2C1731750501869.1731750503389 2024-11-16T09:48:23,416 DEBUG [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:23,521 DEBUG [037c58a19ddb:43181 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T09:48:23,535 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(204): Hosts are {037c58a19ddb=0} racks are {/default-rack=0} 2024-11-16T09:48:23,538 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T09:48:23,538 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T09:48:23,538 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T09:48:23,539 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T09:48:23,539 INFO [037c58a19ddb:43181 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T09:48:23,539 INFO [037c58a19ddb:43181 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T09:48:23,539 DEBUG [037c58a19ddb:43181 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T09:48:23,546 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,552 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,42857,1731750501869, state=OPENING 2024-11-16T09:48:23,557 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T09:48:23,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:23,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:23,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:23,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,560 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,562 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:23,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,42857,1731750501869}] 2024-11-16T09:48:23,738 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:48:23,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34497, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:48:23,755 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T09:48:23,755 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:23,756 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T09:48:23,757 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C42857%2C1731750501869.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,42857,1731750501869, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:23,779 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,42857,1731750501869/037c58a19ddb%2C42857%2C1731750501869.meta.1731750503759.meta 2024-11-16T09:48:23,780 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:23,780 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:23,783 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T09:48:23,787 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T09:48:23,793 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T09:48:23,798 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T09:48:23,798 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:23,798 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T09:48:23,799 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T09:48:23,800 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T09:48:23,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:48:23,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,801 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T09:48:23,803 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:48:23,803 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,803 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T09:48:23,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:48:23,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,804 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T09:48:23,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:48:23,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:23,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:23,806 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:48:23,807 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:23,808 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:23,809 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:48:23,809 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:48:23,809 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:23,811 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:48:23,811 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67729210, jitterRate=0.009243875741958618}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:23,812 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T09:48:23,813 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731750503799Writing region info on filesystem at 1731750503799Initializing all the Stores at 1731750503800 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503800Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503800Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750503800Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750503800Cleaning up temporary data from old regions at 1731750503809 (+9 ms)Running coprocessor post-open hooks at 1731750503812 (+3 ms)Region opened successfully at 1731750503813 (+1 ms) 2024-11-16T09:48:23,819 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731750503728 2024-11-16T09:48:23,830 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T09:48:23,831 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T09:48:23,831 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,833 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,42857,1731750501869, state=OPEN 2024-11-16T09:48:23,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:23,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:23,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:23,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,836 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:23,836 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:23,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T09:48:23,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,42857,1731750501869 in 272 msec 2024-11-16T09:48:23,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T09:48:23,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec 2024-11-16T09:48:23,845 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:23,845 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T09:48:23,868 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:48:23,870 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,42857,1731750501869, seqNum=-1] 2024-11-16T09:48:23,894 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:23,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:23,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0420 sec 2024-11-16T09:48:23,915 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731750503915, completionTime=-1 2024-11-16T09:48:23,918 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T09:48:23,919 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T09:48:23,945 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T09:48:23,945 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750563945 2024-11-16T09:48:23,945 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750623945 2024-11-16T09:48:23,946 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-16T09:48:23,948 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T09:48:23,954 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,954 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,954 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,956 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-037c58a19ddb:43181, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,956 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,956 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:23,963 DEBUG [master/037c58a19ddb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T09:48:23,984 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.885sec 2024-11-16T09:48:23,985 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T09:48:23,987 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T09:48:23,988 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T09:48:23,989 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T09:48:23,989 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T09:48:23,990 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:23,991 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T09:48:23,992 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T09:48:23,993 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T09:48:23,994 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,43181,1731750501093-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:24,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5802b066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:24,047 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T09:48:24,047 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T09:48:24,052 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,43181,-1 for getting cluster id 2024-11-16T09:48:24,056 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:48:24,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '650bf649-60a7-42f3-86e3-14fb1f857542' 2024-11-16T09:48:24,067 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:48:24,068 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "650bf649-60a7-42f3-86e3-14fb1f857542" 2024-11-16T09:48:24,068 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b3594b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:24,068 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,43181,-1] 2024-11-16T09:48:24,071 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:48:24,073 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:24,074 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:48:24,076 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a596f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:24,077 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:48:24,084 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,42857,1731750501869, seqNum=-1] 2024-11-16T09:48:24,085 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:24,087 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:24,108 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=037c58a19ddb,43181,1731750501093 2024-11-16T09:48:24,112 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:48:24,133 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,43181,1731750501093 2024-11-16T09:48:24,136 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18b8acd5 2024-11-16T09:48:24,137 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:48:24,139 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:48:24,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731750504108', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:48:24,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 2024-11-16T09:48:24,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:48:24,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731750504108" procId is: 4 2024-11-16T09:48:24,159 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:24,161 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:48:24,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:24,183 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 116fc12507fbd9b3448eb15aa5ab7b5b, NAME => 'testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731750504108', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1722): Closing 116fc12507fbd9b3448eb15aa5ab7b5b, disabling compactions & flushes 2024-11-16T09:48:24,190 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. after waiting 0 ms 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,190 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,190 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731750504108-pool-0 {}] regionserver.HRegion(1676): Region close journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Waiting for close lock at 1731750504190Disabling compacts and flushes for region at 1731750504190Disabling writes for close at 1731750504190Writing region close event to WAL at 1731750504190Closed at 1731750504190 2024-11-16T09:48:24,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:48:24,197 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731750504192"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750504192"}]},"ts":"1731750504192"} 2024-11-16T09:48:24,201 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:48:24,202 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:48:24,205 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731750504108","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750504202"}]},"ts":"1731750504202"} 2024-11-16T09:48:24,209 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731750504108, state=ENABLING in hbase:meta 2024-11-16T09:48:24,210 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {037c58a19ddb=0} racks are {/default-rack=0} 2024-11-16T09:48:24,211 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T09:48:24,211 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T09:48:24,211 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T09:48:24,211 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T09:48:24,211 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T09:48:24,211 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T09:48:24,211 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T09:48:24,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, ASSIGN}] 2024-11-16T09:48:24,215 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, ASSIGN 2024-11-16T09:48:24,216 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, ASSIGN; state=OFFLINE, location=037c58a19ddb,44683,1731750501996; forceNewPlan=false, retain=false 2024-11-16T09:48:24,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:24,369 INFO [037c58a19ddb:43181 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T09:48:24,370 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=OPENING, regionLocation=037c58a19ddb,44683,1731750501996 2024-11-16T09:48:24,374 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, ASSIGN because future has completed 2024-11-16T09:48:24,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996}] 2024-11-16T09:48:24,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:24,529 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:48:24,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36363, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:48:24,533 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,533 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 116fc12507fbd9b3448eb15aa5ab7b5b, NAME => 'testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:24,534 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731750504108 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,534 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:24,534 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,534 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,535 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,541 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116fc12507fbd9b3448eb15aa5ab7b5b columnFamilyName family 2024-11-16T09:48:24,541 DEBUG [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:24,541 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(327): Store=116fc12507fbd9b3448eb15aa5ab7b5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:24,542 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,542 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,542 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,543 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,543 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,544 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,552 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:24,552 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 116fc12507fbd9b3448eb15aa5ab7b5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60618923, jitterRate=-0.09670765697956085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T09:48:24,553 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:24,554 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Running coprocessor pre-open hook at 1731750504535Writing region info on filesystem at 1731750504535Initializing all the Stores at 1731750504535Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750504535Cleaning up temporary data from old regions at 1731750504543 (+8 ms)Running coprocessor post-open hooks at 1731750504553 (+10 ms)Region opened successfully at 1731750504554 (+1 ms) 2024-11-16T09:48:24,555 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b., pid=6, masterSystemTime=1731750504528 2024-11-16T09:48:24,559 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,559 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:24,559 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,44683,1731750501996 2024-11-16T09:48:24,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996 because future has completed 2024-11-16T09:48:24,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T09:48:24,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996 in 190 msec 2024-11-16T09:48:24,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T09:48:24,569 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, ASSIGN in 354 msec 2024-11-16T09:48:24,571 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:48:24,571 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731750504108","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750504571"}]},"ts":"1731750504571"} 2024-11-16T09:48:24,574 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731750504108, state=ENABLED in hbase:meta 2024-11-16T09:48:24,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:48:24,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731750504108 in 426 msec 2024-11-16T09:48:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:24,791 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731750504108 completed 2024-11-16T09:48:24,792 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731750504108 get assigned. Timeout = 60000ms 2024-11-16T09:48:24,794 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T09:48:24,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731750504108 assigned to meta. Checking AM states. 2024-11-16T09:48:24,801 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T09:48:24,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731750504108 assigned. 2024-11-16T09:48:24,810 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731750504108', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b., hostname=037c58a19ddb,44683,1731750501996, seqNum=2] 2024-11-16T09:48:24,812 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:24,814 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:24,823 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 116fc12507fbd9b3448eb15aa5ab7b5b 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T09:48:24,848 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b15ab58708b44f50be749c293789c489 2024-11-16T09:48:24,885 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/.tmp/family/b15ab58708b44f50be749c293789c489 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T09:48:24,890 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/.tmp/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:24,924 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/.tmp/family/b15ab58708b44f50be749c293789c489 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:24,926 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, delay=933 ms 2024-11-16T09:48:24,927 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T09:48:24,933 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 116fc12507fbd9b3448eb15aa5ab7b5b in 109ms, sequenceid=6, compaction requested=false 2024-11-16T09:48:24,934 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731750504108' 2024-11-16T09:48:24,935 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 116fc12507fbd9b3448eb15aa5ab7b5b: 2024-11-16T09:48:25,861 DEBUG [hfile-prefetch-1731750504926 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b15ab58708b44f50be749c293789c489_0. 2024-11-16T09:48:25,865 DEBUG [hfile-prefetch-1731750504926 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b15ab58708b44f50be749c293789c489_117. 2024-11-16T09:48:25,866 DEBUG [hfile-prefetch-1731750504926 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, with total blocks 2, and data blocks 1 2024-11-16T09:48:25,866 DEBUG [hfile-prefetch-1731750504926 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T09:48:25,867 DEBUG [hfile-prefetch-1731750504926 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T09:48:25,867 DEBUG [hfile-prefetch-1731750504926 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489. Will sleep 100ms and try the verification again. 2024-11-16T09:48:25,913 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T09:48:25,913 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T09:48:25,914 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:25,914 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:25,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-16T09:48:25,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731750505913 2024-11-16T09:48:25,934 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T09:48:25,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=116fc12507fbd9b3448eb15aa5ab7b5b, source=037c58a19ddb,44683,1731750501996, destination=037c58a19ddb,42857,1731750501869, warming up region on 037c58a19ddb,42857,1731750501869 2024-11-16T09:48:25,949 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=116fc12507fbd9b3448eb15aa5ab7b5b, source=037c58a19ddb,44683,1731750501996, destination=037c58a19ddb,42857,1731750501869, running balancer 2024-11-16T09:48:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE 2024-11-16T09:48:25,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:25,951 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE 2024-11-16T09:48:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(7855): Warmup {ENCODED => 116fc12507fbd9b3448eb15aa5ab7b5b, NAME => 'testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:25,952 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:25,953 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116fc12507fbd9b3448eb15aa5ab7b5b columnFamilyName family 2024-11-16T09:48:25,953 DEBUG [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:25,953 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=CLOSING, regionLocation=037c58a19ddb,44683,1731750501996 2024-11-16T09:48:25,956 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE because future has completed 2024-11-16T09:48:25,956 DEBUG [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:25,956 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(327): Store=116fc12507fbd9b3448eb15aa5ab7b5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:25,956 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-16T09:48:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1722): Closing 116fc12507fbd9b3448eb15aa5ab7b5b, disabling compactions & flushes 2024-11-16T09:48:25,957 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:25,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996}] 2024-11-16T09:48:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. after waiting 0 ms 2024-11-16T09:48:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:25,959 DEBUG [StoreFileCloser-116fc12507fbd9b3448eb15aa5ab7b5b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:25,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42857 {}] regionserver.HRegion(1676): Region close journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Waiting for close lock at 1731750505956Disabling compacts and flushes for region at 1731750505956Disabling writes for close at 1731750505957 (+1 ms)Writing region close event to WAL at 1731750505960 (+3 ms)Closed at 1731750505960 2024-11-16T09:48:25,962 WARN [hfile-prefetch-1731750504926 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-16T09:48:25,963 DEBUG [hfile-prefetch-1731750504926 {}] hfile.PrefetchExecutor(121): Prefetch completed for b15ab58708b44f50be749c293789c489 2024-11-16T09:48:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43181 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T09:48:26,114 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,115 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-16T09:48:26,115 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 116fc12507fbd9b3448eb15aa5ab7b5b, disabling compactions & flushes 2024-11-16T09:48:26,115 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,116 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,116 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. after waiting 0 ms 2024-11-16T09:48:26,116 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,117 DEBUG [StoreFileCloser-116fc12507fbd9b3448eb15aa5ab7b5b-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b15ab58708b44f50be749c293789c489, starting offset: 0, end offset: 9223372036854775807 2024-11-16T09:48:26,119 DEBUG [StoreFileCloser-116fc12507fbd9b3448eb15aa5ab7b5b-family-1 {}] bucket.BucketCache(856): removed key b15ab58708b44f50be749c293789c489_0 from back map with offset lock 2094080 in the evict process 2024-11-16T09:48:26,124 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T09:48:26,125 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,126 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Waiting for close lock at 1731750506115Running coprocessor pre-close hooks at 1731750506115Disabling compacts and flushes for region at 1731750506115Disabling writes for close at 1731750506116 (+1 ms)Writing region close event to WAL at 1731750506119 (+3 ms)Running coprocessor post-close hooks at 1731750506124 (+5 ms)Closed at 1731750506125 (+1 ms) 2024-11-16T09:48:26,126 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 116fc12507fbd9b3448eb15aa5ab7b5b move to 037c58a19ddb,42857,1731750501869 record at close sequenceid=6 2024-11-16T09:48:26,129 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,130 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=CLOSED 2024-11-16T09:48:26,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996 because future has completed 2024-11-16T09:48:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T09:48:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T09:48:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:26,134 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:26,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T09:48:26,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,44683,1731750501996 in 177 msec 2024-11-16T09:48:26,137 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE; state=CLOSED, location=037c58a19ddb,42857,1731750501869; forceNewPlan=false, retain=false 2024-11-16T09:48:26,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:26,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750506134 2024-11-16T09:48:26,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T09:48:26,287 INFO [037c58a19ddb:43181 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T09:48:26,288 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=OPENING, regionLocation=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:26,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE because future has completed 2024-11-16T09:48:26,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,42857,1731750501869}] 2024-11-16T09:48:26,447 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,447 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 116fc12507fbd9b3448eb15aa5ab7b5b, NAME => 'testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:26,447 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731750504108 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:26,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,448 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,449 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,450 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 116fc12507fbd9b3448eb15aa5ab7b5b columnFamilyName family 2024-11-16T09:48:26,450 DEBUG [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:26,452 DEBUG [StoreFileOpener-116fc12507fbd9b3448eb15aa5ab7b5b-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489, delay=939 ms 2024-11-16T09:48:26,453 DEBUG [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:26,453 INFO [StoreOpener-116fc12507fbd9b3448eb15aa5ab7b5b-1 {}] regionserver.HStore(327): Store=116fc12507fbd9b3448eb15aa5ab7b5b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:26,454 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,454 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,455 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,455 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,455 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,460 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,460 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 116fc12507fbd9b3448eb15aa5ab7b5b; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62867305, jitterRate=-0.06320415437221527}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T09:48:26,461 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,462 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Running coprocessor pre-open hook at 1731750506448Writing region info on filesystem at 1731750506448Initializing all the Stores at 1731750506448Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750506448Cleaning up temporary data from old regions at 1731750506455 (+7 ms)Running coprocessor post-open hooks at 1731750506461 (+6 ms)Region opened successfully at 1731750506462 (+1 ms) 2024-11-16T09:48:26,463 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b., pid=9, masterSystemTime=1731750506444 2024-11-16T09:48:26,465 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,466 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,466 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=116fc12507fbd9b3448eb15aa5ab7b5b, regionState=OPEN, openSeqNum=10, regionLocation=037c58a19ddb,42857,1731750501869 2024-11-16T09:48:26,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,42857,1731750501869 because future has completed 2024-11-16T09:48:26,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T09:48:26,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 116fc12507fbd9b3448eb15aa5ab7b5b, server=037c58a19ddb,42857,1731750501869 in 178 msec 2024-11-16T09:48:26,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731750504108, region=116fc12507fbd9b3448eb15aa5ab7b5b, REOPEN/MOVE in 523 msec 2024-11-16T09:48:26,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T09:48:26,504 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:48:26,504 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:26,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,510 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:48:26,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T09:48:26,511 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-16T09:48:26,511 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=037c58a19ddb,43181,1731750501093 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:26,514 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T09:48:26,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:26,514 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:48:26,515 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:26,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:26,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:26,516 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,42857,1731750501869' ***** 2024-11-16T09:48:26,516 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T09:48:26,516 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:26,516 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,44683,1731750501996' ***** 2024-11-16T09:48:26,516 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T09:48:26,516 INFO [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T09:48:26,516 INFO [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T09:48:26,516 INFO [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence 2024-11-16T09:48:26,516 INFO [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/bucket.persistence 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T09:48:26,517 INFO [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T09:48:26,517 INFO [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T09:48:26,517 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T09:48:26,517 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T09:48:26,527 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1487): Thread RS:0;037c58a19ddb:42857 started persisting bucket cache to file 2024-11-16T09:48:26,527 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1487): Thread RS:1;037c58a19ddb:44683 started persisting bucket cache to file 2024-11-16T09:48:26,527 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:26,527 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:26,527 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:26,527 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:26,531 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:26,531 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750506527 2024-11-16T09:48:26,531 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:26,531 DEBUG [RS:1;037c58a19ddb:44683 {}] bucket.BucketCache(1509): Thread RS:1;037c58a19ddb:44683 finished persisting bucket cache to file, renaming 2024-11-16T09:48:26,531 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750506527 2024-11-16T09:48:26,531 DEBUG [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1509): Thread RS:0;037c58a19ddb:42857 finished persisting bucket cache to file, renaming 2024-11-16T09:48:26,531 WARN [RS:0;037c58a19ddb:42857 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-16T09:48:26,531 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T09:48:26,531 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T09:48:26,532 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T09:48:26,532 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T09:48:26,532 INFO [RS:0;037c58a19ddb:42857 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T09:48:26,532 INFO [RS:1;037c58a19ddb:44683 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T09:48:26,532 INFO [RS:0;037c58a19ddb:42857 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T09:48:26,532 INFO [RS:1;037c58a19ddb:44683 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T09:48:26,532 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(3091): Received CLOSE for 116fc12507fbd9b3448eb15aa5ab7b5b 2024-11-16T09:48:26,532 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,44683,1731750501996 2024-11-16T09:48:26,532 INFO [RS:1;037c58a19ddb:44683 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:26,532 INFO [RS:1;037c58a19ddb:44683 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;037c58a19ddb:44683. 2024-11-16T09:48:26,532 DEBUG [RS:1;037c58a19ddb:44683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:26,532 DEBUG [RS:1;037c58a19ddb:44683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,42857,1731750501869 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:26,533 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,44683,1731750501996; all regions closed. 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;037c58a19ddb:42857. 2024-11-16T09:48:26,533 DEBUG [RS:0;037c58a19ddb:42857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:26,533 DEBUG [RS:0;037c58a19ddb:42857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,533 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T09:48:26,533 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T09:48:26,533 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 116fc12507fbd9b3448eb15aa5ab7b5b, disabling compactions & flushes 2024-11-16T09:48:26,533 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. after waiting 0 ms 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,534 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T09:48:26,534 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1325): Online Regions={116fc12507fbd9b3448eb15aa5ab7b5b=testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b., 1588230740=hbase:meta,,1.1588230740} 2024-11-16T09:48:26,534 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:48:26,534 DEBUG [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1351): Waiting on 116fc12507fbd9b3448eb15aa5ab7b5b, 1588230740 2024-11-16T09:48:26,534 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:48:26,534 DEBUG [StoreFileCloser-116fc12507fbd9b3448eb15aa5ab7b5b-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/family/b15ab58708b44f50be749c293789c489 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:48:26,534 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T09:48:26,535 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:48:26,535 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-16T09:48:26,538 DEBUG [RS:1;037c58a19ddb:44683 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:26,538 INFO [RS:1;037c58a19ddb:44683 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C44683%2C1731750501996:(num 1731750503388) 2024-11-16T09:48:26,538 DEBUG [RS:1;037c58a19ddb:44683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,538 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:26,538 INFO [RS:1;037c58a19ddb:44683 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:26,539 INFO [RS:1;037c58a19ddb:44683 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:26,539 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T09:48:26,539 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T09:48:26,539 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T09:48:26,539 INFO [RS:1;037c58a19ddb:44683 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:26,540 INFO [RS:1;037c58a19ddb:44683 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44683 2024-11-16T09:48:26,542 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnRegionMove1731750504108/116fc12507fbd9b3448eb15aa5ab7b5b/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T09:48:26,542 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:26,543 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,543 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 116fc12507fbd9b3448eb15aa5ab7b5b: Waiting for close lock at 1731750506533Running coprocessor pre-close hooks at 1731750506533Disabling compacts and flushes for region at 1731750506533Disabling writes for close at 1731750506534 (+1 ms)Writing region close event to WAL at 1731750506535 (+1 ms)Running coprocessor post-close hooks at 1731750506543 (+8 ms)Closed at 1731750506543 2024-11-16T09:48:26,543 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b. 2024-11-16T09:48:26,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,44683,1731750501996 2024-11-16T09:48:26,545 INFO [RS:1;037c58a19ddb:44683 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:26,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:26,546 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,44683,1731750501996] 2024-11-16T09:48:26,547 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fd616dc088984431b5dd2501838f461f 2024-11-16T09:48:26,548 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,44683,1731750501996 already deleted, retry=false 2024-11-16T09:48:26,548 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,44683,1731750501996 expired; onlineServers=1 2024-11-16T09:48:26,565 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/fd616dc088984431b5dd2501838f461f is 205, key is testBlockEvictionOnRegionMove1731750504108,,1731750504141.116fc12507fbd9b3448eb15aa5ab7b5b./info:regioninfo/1731750506466/Put/seqid=0 2024-11-16T09:48:26,566 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/fd616dc088984431b5dd2501838f461f 2024-11-16T09:48:26,575 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1268dab538ad405c91a7d16420c3b5a4 2024-11-16T09:48:26,588 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/1268dab538ad405c91a7d16420c3b5a4 is 43, key is default/ns:d/1731750503901/Put/seqid=0 2024-11-16T09:48:26,589 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/1268dab538ad405c91a7d16420c3b5a4 2024-11-16T09:48:26,598 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 37a5612a0d3648e9b1bf91b0e56711c4 2024-11-16T09:48:26,608 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:26,611 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:26,612 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/37a5612a0d3648e9b1bf91b0e56711c4 is 78, key is testBlockEvictionOnRegionMove1731750504108/table:state/1731750504571/Put/seqid=0 2024-11-16T09:48:26,613 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/37a5612a0d3648e9b1bf91b0e56711c4 2024-11-16T09:48:26,615 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/fd616dc088984431b5dd2501838f461f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/fd616dc088984431b5dd2501838f461f 2024-11-16T09:48:26,616 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/fd616dc088984431b5dd2501838f461f, delay=928 ms 2024-11-16T09:48:26,618 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/fd616dc088984431b5dd2501838f461f, entries=15, sequenceid=15, filesize=7.6 K 2024-11-16T09:48:26,618 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/1268dab538ad405c91a7d16420c3b5a4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/1268dab538ad405c91a7d16420c3b5a4 2024-11-16T09:48:26,619 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/1268dab538ad405c91a7d16420c3b5a4, delay=935 ms 2024-11-16T09:48:26,621 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/1268dab538ad405c91a7d16420c3b5a4, entries=2, sequenceid=15, filesize=5.0 K 2024-11-16T09:48:26,621 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/37a5612a0d3648e9b1bf91b0e56711c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/37a5612a0d3648e9b1bf91b0e56711c4 2024-11-16T09:48:26,622 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/37a5612a0d3648e9b1bf91b0e56711c4, delay=954 ms 2024-11-16T09:48:26,624 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/37a5612a0d3648e9b1bf91b0e56711c4, entries=2, sequenceid=15, filesize=5.3 K 2024-11-16T09:48:26,625 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-11-16T09:48:26,625 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T09:48:26,626 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/fd616dc088984431b5dd2501838f461f 2024-11-16T09:48:26,627 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/1268dab538ad405c91a7d16420c3b5a4 2024-11-16T09:48:26,628 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/37a5612a0d3648e9b1bf91b0e56711c4 2024-11-16T09:48:26,633 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-16T09:48:26,634 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T09:48:26,634 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:26,635 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731750506534Running coprocessor pre-close hooks at 1731750506534Disabling compacts and flushes for region at 1731750506534Disabling writes for close at 1731750506534Obtaining lock to block concurrent updates at 1731750506535 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731750506535Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731750506536 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731750506536Flushing 1588230740/info: creating writer at 1731750506536Flushing 1588230740/info: appending metadata at 1731750506563 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731750506563Flushing 1588230740/ns: creating writer at 1731750506568 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731750506587 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1731750506587Flushing 1588230740/table: creating writer at 1731750506591 (+4 ms)Flushing 1588230740/table: appending metadata at 1731750506611 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731750506611Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74bacf5e: reopening flushed file at 1731750506615 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1461503c: reopening flushed file at 1731750506618 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7aecf63e: reopening flushed file at 1731750506621 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1731750506625 (+4 ms)Writing region close event to WAL at 1731750506628 (+3 ms)Running coprocessor post-close hooks at 1731750506634 (+6 ms)Closed at 1731750506634 2024-11-16T09:48:26,635 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:26,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44683-0x101b07b0f070002, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,648 INFO [RS:1;037c58a19ddb:44683 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:26,649 INFO [RS:1;037c58a19ddb:44683 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,44683,1731750501996; zookeeper connection closed. 2024-11-16T09:48:26,649 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3b40e07d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3b40e07d 2024-11-16T09:48:26,734 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,42857,1731750501869; all regions closed. 2024-11-16T09:48:26,737 DEBUG [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:26,737 INFO [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C42857%2C1731750501869.meta:.meta(num 1731750503759) 2024-11-16T09:48:26,739 DEBUG [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:26,739 INFO [RS:0;037c58a19ddb:42857 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C42857%2C1731750501869:(num 1731750503389) 2024-11-16T09:48:26,739 DEBUG [RS:0;037c58a19ddb:42857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:26,739 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:26,739 INFO [RS:0;037c58a19ddb:42857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:26,739 INFO [RS:0;037c58a19ddb:42857 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:26,739 INFO [RS:0;037c58a19ddb:42857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:26,740 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:26,740 INFO [RS:0;037c58a19ddb:42857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42857 2024-11-16T09:48:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,42857,1731750501869 2024-11-16T09:48:26,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:26,742 INFO [RS:0;037c58a19ddb:42857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:26,744 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,42857,1731750501869] 2024-11-16T09:48:26,746 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,42857,1731750501869 already deleted, retry=false 2024-11-16T09:48:26,746 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,42857,1731750501869 expired; onlineServers=0 2024-11-16T09:48:26,746 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '037c58a19ddb,43181,1731750501093' ***** 2024-11-16T09:48:26,746 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T09:48:26,746 INFO [M:0;037c58a19ddb:43181 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:26,746 INFO [M:0;037c58a19ddb:43181 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:26,746 DEBUG [M:0;037c58a19ddb:43181 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T09:48:26,746 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T09:48:26,746 DEBUG [M:0;037c58a19ddb:43181 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T09:48:26,746 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750503000 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750503000,5,FailOnTimeoutGroup] 2024-11-16T09:48:26,746 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750502996 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750502996,5,FailOnTimeoutGroup] 2024-11-16T09:48:26,747 INFO [M:0;037c58a19ddb:43181 {}] hbase.ChoreService(370): Chore service for: master/037c58a19ddb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:26,747 INFO [M:0;037c58a19ddb:43181 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:26,747 DEBUG [M:0;037c58a19ddb:43181 {}] master.HMaster(1795): Stopping service threads 2024-11-16T09:48:26,747 INFO [M:0;037c58a19ddb:43181 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T09:48:26,747 INFO [M:0;037c58a19ddb:43181 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T09:48:26,748 INFO [M:0;037c58a19ddb:43181 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T09:48:26,748 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T09:48:26,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:26,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:26,749 DEBUG [M:0;037c58a19ddb:43181 {}] zookeeper.ZKUtil(347): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T09:48:26,749 WARN [M:0;037c58a19ddb:43181 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T09:48:26,750 INFO [M:0;037c58a19ddb:43181 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.lastflushedseqids 2024-11-16T09:48:26,765 INFO [M:0;037c58a19ddb:43181 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T09:48:26,765 INFO [M:0;037c58a19ddb:43181 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T09:48:26,765 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T09:48:26,765 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:26,765 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:26,765 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T09:48:26,765 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:26,766 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-16T09:48:26,777 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b852d48bd9e1472d9a16f82394c41c51 2024-11-16T09:48:26,797 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b852d48bd9e1472d9a16f82394c41c51 is 82, key is hbase:meta,,1/info:regioninfo/1731750503831/Put/seqid=0 2024-11-16T09:48:26,798 INFO [M:0;037c58a19ddb:43181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b852d48bd9e1472d9a16f82394c41c51 2024-11-16T09:48:26,808 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b1167300209348ed9f257be33760ea9e 2024-11-16T09:48:26,821 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1167300209348ed9f257be33760ea9e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731750504578/Put/seqid=0 2024-11-16T09:48:26,821 INFO [M:0;037c58a19ddb:43181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1167300209348ed9f257be33760ea9e 2024-11-16T09:48:26,829 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 30d7f1ec59c6417c8b67e09451e7dec9 2024-11-16T09:48:26,843 DEBUG [M:0;037c58a19ddb:43181 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/30d7f1ec59c6417c8b67e09451e7dec9 is 69, key is 037c58a19ddb,42857,1731750501869/rs:state/1731750503109/Put/seqid=0 2024-11-16T09:48:26,843 INFO [M:0;037c58a19ddb:43181 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/30d7f1ec59c6417c8b67e09451e7dec9 2024-11-16T09:48:26,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,844 INFO [RS:0;037c58a19ddb:42857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:26,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42857-0x101b07b0f070001, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,844 INFO [RS:0;037c58a19ddb:42857 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,42857,1731750501869; zookeeper connection closed. 2024-11-16T09:48:26,845 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@ae179f8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@ae179f8 2024-11-16T09:48:26,845 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-16T09:48:26,846 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b852d48bd9e1472d9a16f82394c41c51 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b852d48bd9e1472d9a16f82394c41c51 2024-11-16T09:48:26,847 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b852d48bd9e1472d9a16f82394c41c51, entries=8, sequenceid=84, filesize=5.5 K 2024-11-16T09:48:26,847 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b1167300209348ed9f257be33760ea9e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b1167300209348ed9f257be33760ea9e 2024-11-16T09:48:26,849 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b1167300209348ed9f257be33760ea9e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-16T09:48:26,849 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/30d7f1ec59c6417c8b67e09451e7dec9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/30d7f1ec59c6417c8b67e09451e7dec9 2024-11-16T09:48:26,851 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/30d7f1ec59c6417c8b67e09451e7dec9, entries=2, sequenceid=84, filesize=5.1 K 2024-11-16T09:48:26,851 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false 2024-11-16T09:48:26,853 INFO [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:26,853 DEBUG [M:0;037c58a19ddb:43181 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731750506765Disabling compacts and flushes for region at 1731750506765Disabling writes for close at 1731750506765Obtaining lock to block concurrent updates at 1731750506766 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731750506766Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731750506766Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731750506767 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731750506767Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731750506796 (+29 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731750506796Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731750506800 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731750506820 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731750506820Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731750506823 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731750506842 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731750506842Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1731750506845 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1731750506847 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1731750506849 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 85ms, sequenceid=84, compaction requested=false at 1731750506851 (+2 ms)Writing region close event to WAL at 1731750506853 (+2 ms)Closed at 1731750506853 2024-11-16T09:48:26,855 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:26,855 INFO [M:0;037c58a19ddb:43181 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T09:48:26,855 INFO [M:0;037c58a19ddb:43181 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43181 2024-11-16T09:48:26,855 INFO [M:0;037c58a19ddb:43181 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:26,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,957 INFO [M:0;037c58a19ddb:43181 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:26,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43181-0x101b07b0f070000, quorum=127.0.0.1:59347, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:26,962 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T09:48:26,984 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T09:48:26,984 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:26,993 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/037c58a19ddb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/037c58a19ddb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731750506620 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/037c58a19ddb:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731750504926 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731750506617 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/037c58a19ddb:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731750506452 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=613 (was 623), ProcessCount=11 (was 11), AvailableMemoryMB=6723 (was 7099) 2024-11-16T09:48:27,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=613, ProcessCount=11, AvailableMemoryMB=6723 2024-11-16T09:48:27,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/hadoop-log-dir so I do NOT create it in target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f 2024-11-16T09:48:27,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/hadoop-tmp-dir so I do NOT create it in target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f 2024-11-16T09:48:27,000 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,003 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/cluster_c811c978-3c77-419f-dd5e-af8af658b8c0, deleteOnExit=true 2024-11-16T09:48:27,006 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/cluster_c811c978-3c77-419f-dd5e-af8af658b8c0/zookeeper_0, clientPort=55936, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/cluster_c811c978-3c77-419f-dd5e-af8af658b8c0/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/cluster_c811c978-3c77-419f-dd5e-af8af658b8c0/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T09:48:27,007 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55936 2024-11-16T09:48:27,023 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 with version=8 2024-11-16T09:48:27,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase-staging 2024-11-16T09:48:27,025 INFO [Time-limited test {}] client.ConnectionUtils(128): master/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:27,025 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T09:48:27,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:27,027 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46839 2024-11-16T09:48:27,028 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46839 connecting to ZooKeeper ensemble=127.0.0.1:55936 2024-11-16T09:48:27,035 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:468390x0, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:27,035 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46839-0x101b07b27660000 connected 2024-11-16T09:48:27,058 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,058 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,061 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:27,061 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33, hbase.cluster.distributed=false 2024-11-16T09:48:27,063 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:27,063 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46839 2024-11-16T09:48:27,064 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46839 2024-11-16T09:48:27,067 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46839 2024-11-16T09:48:27,068 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46839 2024-11-16T09:48:27,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46839 2024-11-16T09:48:27,086 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:27,086 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:48:27,087 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:27,088 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35201 2024-11-16T09:48:27,089 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35201 connecting to ZooKeeper ensemble=127.0.0.1:55936 2024-11-16T09:48:27,089 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,090 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,093 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:352010x0, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:27,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35201-0x101b07b27660001 connected 2024-11-16T09:48:27,093 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:27,094 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:48:27,095 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache 2024-11-16T09:48:27,095 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T09:48:27,095 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T09:48:27,095 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T09:48:27,095 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T09:48:27,095 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T09:48:27,095 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T09:48:27,096 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T09:48:27,096 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence is complete. 2024-11-16T09:48:27,096 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:48:27,097 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T09:48:27,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:27,098 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35201 2024-11-16T09:48:27,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35201 2024-11-16T09:48:27,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35201 2024-11-16T09:48:27,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35201 2024-11-16T09:48:27,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35201 2024-11-16T09:48:27,118 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:27,118 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:48:27,119 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:27,120 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41283 2024-11-16T09:48:27,122 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41283 connecting to ZooKeeper ensemble=127.0.0.1:55936 2024-11-16T09:48:27,122 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,123 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412830x0, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:27,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41283-0x101b07b27660002 connected 2024-11-16T09:48:27,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:27,127 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:48:27,127 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache 2024-11-16T09:48:27,127 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T09:48:27,128 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T09:48:27,128 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T09:48:27,128 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T09:48:27,128 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T09:48:27,128 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T09:48:27,128 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:48:27,128 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T09:48:27,129 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence is complete. 2024-11-16T09:48:27,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T09:48:27,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:27,131 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41283 2024-11-16T09:48:27,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41283 2024-11-16T09:48:27,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41283 2024-11-16T09:48:27,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41283 2024-11-16T09:48:27,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41283 2024-11-16T09:48:27,158 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;037c58a19ddb:46839 2024-11-16T09:48:27,159 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,162 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:27,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:27,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,165 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T09:48:27,165 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/037c58a19ddb,46839,1731750507025 from backup master directory 2024-11-16T09:48:27,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T09:48:27,167 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:27,167 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,169 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase.id] with ID: 5f2775f2-0950-4ccb-b368-ac3a577fc128 2024-11-16T09:48:27,169 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.tmp/hbase.id 2024-11-16T09:48:27,172 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/hbase.id] 2024-11-16T09:48:27,210 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:27,210 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T09:48:27,212 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-16T09:48:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,220 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:48:27,222 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T09:48:27,222 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:27,244 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store 2024-11-16T09:48:27,250 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T09:48:27,251 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:27,251 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731750507250Disabling compacts and flushes for region at 1731750507250Disabling writes for close at 1731750507251 (+1 ms)Writing region close event to WAL at 1731750507251Closed at 1731750507251 2024-11-16T09:48:27,251 WARN [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/.initializing 2024-11-16T09:48:27,251 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,257 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:27,262 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=037c58a19ddb%2C46839%2C1731750507025, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,46839,1731750507025, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/oldWALs, maxLogs=10 2024-11-16T09:48:27,281 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/WALs/037c58a19ddb,46839,1731750507025/037c58a19ddb%2C46839%2C1731750507025.1731750507262 2024-11-16T09:48:27,281 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:27,282 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:27,282 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:27,282 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,282 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,283 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T09:48:27,288 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:27,288 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,292 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T09:48:27,293 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:27,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T09:48:27,297 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:27,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T09:48:27,301 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:27,301 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,302 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,302 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,303 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,303 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,303 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:27,304 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T09:48:27,310 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:27,310 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69473092, jitterRate=0.03522974252700806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:27,312 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731750507282Initializing all the Stores at 1731750507282Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750507283 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750507283Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750507283Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750507283Cleaning up temporary data from old regions at 1731750507303 (+20 ms)Region opened successfully at 1731750507311 (+8 ms) 2024-11-16T09:48:27,312 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T09:48:27,317 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ae4e77f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:27,318 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T09:48:27,318 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T09:48:27,318 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T09:48:27,318 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T09:48:27,319 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T09:48:27,319 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T09:48:27,319 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T09:48:27,322 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T09:48:27,323 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T09:48:27,325 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T09:48:27,325 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T09:48:27,326 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T09:48:27,327 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T09:48:27,327 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T09:48:27,328 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T09:48:27,330 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T09:48:27,331 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T09:48:27,332 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T09:48:27,335 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T09:48:27,336 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,339 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=037c58a19ddb,46839,1731750507025, sessionid=0x101b07b27660000, setting cluster-up flag (Was=false) 2024-11-16T09:48:27,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,348 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T09:48:27,350 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,358 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T09:48:27,359 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=037c58a19ddb,46839,1731750507025 2024-11-16T09:48:27,360 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T09:48:27,361 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:27,361 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T09:48:27,361 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T09:48:27,362 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 037c58a19ddb,46839,1731750507025 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/037c58a19ddb:0, corePoolSize=5, maxPoolSize=5 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/037c58a19ddb:0, corePoolSize=10, maxPoolSize=10 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:27,364 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,366 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:27,366 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T09:48:27,367 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,368 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731750537368 2024-11-16T09:48:27,367 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T09:48:27,368 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T09:48:27,368 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T09:48:27,368 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T09:48:27,369 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T09:48:27,369 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T09:48:27,369 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T09:48:27,370 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,370 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T09:48:27,370 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T09:48:27,370 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T09:48:27,371 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T09:48:27,371 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T09:48:27,376 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750507371,5,FailOnTimeoutGroup] 2024-11-16T09:48:27,378 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750507376,5,FailOnTimeoutGroup] 2024-11-16T09:48:27,378 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,378 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T09:48:27,378 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,378 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,395 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T09:48:27,395 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:27,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:27,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T09:48:27,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:48:27,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:27,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T09:48:27,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:48:27,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:27,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T09:48:27,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:48:27,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:27,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T09:48:27,418 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:48:27,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:27,419 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:27,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:48:27,419 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:27,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:27,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:48:27,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:48:27,420 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:27,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:48:27,428 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:27,428 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69518841, jitterRate=0.03591145575046539}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:27,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731750507402Initializing all the Stores at 1731750507403 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750507403Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750507403Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750507403Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750507403Cleaning up temporary data from old regions at 1731750507420 (+17 ms)Region opened successfully at 1731750507429 (+9 ms) 2024-11-16T09:48:27,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:48:27,429 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:48:27,429 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:48:27,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 1 ms 2024-11-16T09:48:27,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:48:27,430 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:27,430 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731750507429Disabling compacts and flushes for region at 1731750507429Disabling writes for close at 1731750507430 (+1 ms)Writing region close event to WAL at 1731750507430Closed at 1731750507430 2024-11-16T09:48:27,431 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:27,431 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T09:48:27,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T09:48:27,432 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:27,433 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T09:48:27,442 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(746): ClusterId : 5f2775f2-0950-4ccb-b368-ac3a577fc128 2024-11-16T09:48:27,442 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:48:27,442 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(746): ClusterId : 5f2775f2-0950-4ccb-b368-ac3a577fc128 2024-11-16T09:48:27,443 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:48:27,446 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:48:27,446 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:48:27,447 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:48:27,447 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:48:27,450 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:48:27,450 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:48:27,451 DEBUG [RS:0;037c58a19ddb:35201 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2165faab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:27,451 DEBUG [RS:1;037c58a19ddb:41283 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77510027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:27,468 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;037c58a19ddb:35201 2024-11-16T09:48:27,468 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:48:27,468 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:48:27,468 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:48:27,469 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;037c58a19ddb:41283 2024-11-16T09:48:27,469 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:48:27,469 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:48:27,469 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:48:27,469 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,46839,1731750507025 with port=35201, startcode=1731750507086 2024-11-16T09:48:27,470 DEBUG [RS:0;037c58a19ddb:35201 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:48:27,470 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,46839,1731750507025 with port=41283, startcode=1731750507118 2024-11-16T09:48:27,470 DEBUG [RS:1;037c58a19ddb:41283 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:48:27,474 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:48:27,474 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60447, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:48:27,474 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,475 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,476 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,476 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,477 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:27,477 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T09:48:27,477 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:48:27,478 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:27,478 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T09:48:27,478 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:48:27,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:27,484 DEBUG [RS:0;037c58a19ddb:35201 {}] zookeeper.ZKUtil(111): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,485 WARN [RS:0;037c58a19ddb:35201 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:27,485 INFO [RS:0;037c58a19ddb:35201 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:27,485 DEBUG [RS:1;037c58a19ddb:41283 {}] zookeeper.ZKUtil(111): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,485 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,485 WARN [RS:1;037c58a19ddb:41283 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:27,485 INFO [RS:1;037c58a19ddb:41283 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:27,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,41283,1731750507118] 2024-11-16T09:48:27,485 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,35201,1731750507086] 2024-11-16T09:48:27,485 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,491 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:48:27,493 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:48:27,493 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:48:27,494 INFO [RS:1;037c58a19ddb:41283 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:48:27,494 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,494 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:48:27,495 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:48:27,495 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:48:27,495 INFO [RS:0;037c58a19ddb:35201 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:48:27,495 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,495 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,496 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,496 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:27,496 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,496 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:27,497 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:27,497 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 DEBUG [RS:1;037c58a19ddb:41283 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,498 DEBUG [RS:0;037c58a19ddb:35201 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,498 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,41283,1731750507118-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,504 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,35201,1731750507086-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:27,520 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:48:27,520 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,41283,1731750507118-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,520 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,521 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.Replication(171): 037c58a19ddb,41283,1731750507118 started 2024-11-16T09:48:27,529 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:48:27,529 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,35201,1731750507086-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,529 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,529 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.Replication(171): 037c58a19ddb,35201,1731750507086 started 2024-11-16T09:48:27,542 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,542 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,41283,1731750507118, RpcServer on 037c58a19ddb/172.17.0.2:41283, sessionid=0x101b07b27660002 2024-11-16T09:48:27,542 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:48:27,542 DEBUG [RS:1;037c58a19ddb:41283 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,543 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,41283,1731750507118' 2024-11-16T09:48:27,543 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,41283,1731750507118' 2024-11-16T09:48:27,544 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:48:27,545 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:48:27,546 DEBUG [RS:1;037c58a19ddb:41283 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:48:27,546 INFO [RS:1;037c58a19ddb:41283 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:48:27,546 INFO [RS:1;037c58a19ddb:41283 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:48:27,548 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:27,548 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,35201,1731750507086, RpcServer on 037c58a19ddb/172.17.0.2:35201, sessionid=0x101b07b27660001 2024-11-16T09:48:27,548 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:48:27,548 DEBUG [RS:0;037c58a19ddb:35201 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,548 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,35201,1731750507086' 2024-11-16T09:48:27,548 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:48:27,549 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,35201,1731750507086 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,35201,1731750507086' 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:48:27,550 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:48:27,551 DEBUG [RS:0;037c58a19ddb:35201 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:48:27,551 INFO [RS:0;037c58a19ddb:35201 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:48:27,551 INFO [RS:0;037c58a19ddb:35201 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:48:27,583 WARN [037c58a19ddb:46839 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T09:48:27,647 INFO [RS:1;037c58a19ddb:41283 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:27,647 INFO [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C41283%2C1731750507118, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:27,652 INFO [RS:0;037c58a19ddb:35201 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:27,652 INFO [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C35201%2C1731750507086, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,35201,1731750507086, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:27,673 INFO [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118/037c58a19ddb%2C41283%2C1731750507118.1731750507649 2024-11-16T09:48:27,673 DEBUG [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:27,676 INFO [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,35201,1731750507086/037c58a19ddb%2C35201%2C1731750507086.1731750507654 2024-11-16T09:48:27,676 DEBUG [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:27,834 DEBUG [037c58a19ddb:46839 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T09:48:27,834 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(204): Hosts are {037c58a19ddb=0} racks are {/default-rack=0} 2024-11-16T09:48:27,835 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T09:48:27,835 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T09:48:27,835 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T09:48:27,835 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T09:48:27,835 INFO [037c58a19ddb:46839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T09:48:27,835 INFO [037c58a19ddb:46839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T09:48:27,835 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T09:48:27,835 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:27,836 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,41283,1731750507118, state=OPENING 2024-11-16T09:48:27,838 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T09:48:27,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:27,841 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:27,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:27,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:27,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,41283,1731750507118}] 2024-11-16T09:48:27,841 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:27,994 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:48:27,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:48:27,999 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T09:48:27,999 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:28,000 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T09:48:28,001 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C41283%2C1731750507118.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:28,021 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118/037c58a19ddb%2C41283%2C1731750507118.meta.1731750508002.meta 2024-11-16T09:48:28,024 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:28,024 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:28,025 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T09:48:28,025 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T09:48:28,025 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T09:48:28,025 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T09:48:28,025 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:28,025 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T09:48:28,026 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T09:48:28,027 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T09:48:28,028 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:48:28,028 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:28,028 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T09:48:28,029 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:48:28,029 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,029 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:28,030 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T09:48:28,030 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:48:28,030 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:28,031 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T09:48:28,031 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:48:28,031 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,032 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:28,032 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:48:28,032 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:28,033 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:28,033 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:48:28,033 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:48:28,033 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:28,034 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:48:28,034 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68115104, jitterRate=0.014994144439697266}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:28,035 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T09:48:28,035 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731750508026Writing region info on filesystem at 1731750508026Initializing all the Stores at 1731750508026Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750508026Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750508026Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750508026Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750508026Cleaning up temporary data from old regions at 1731750508033 (+7 ms)Running coprocessor post-open hooks at 1731750508035 (+2 ms)Region opened successfully at 1731750508035 2024-11-16T09:48:28,036 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731750507994 2024-11-16T09:48:28,039 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T09:48:28,039 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T09:48:28,040 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:28,041 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,41283,1731750507118, state=OPEN 2024-11-16T09:48:28,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:28,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:28,043 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:28,043 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:28,043 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:28,043 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:28,043 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:28,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T09:48:28,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,41283,1731750507118 in 202 msec 2024-11-16T09:48:28,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T09:48:28,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 618 msec 2024-11-16T09:48:28,051 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T09:48:28,052 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T09:48:28,055 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:48:28,055 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1] 2024-11-16T09:48:28,056 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:28,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35963, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:28,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 702 msec 2024-11-16T09:48:28,064 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731750508064, completionTime=-1 2024-11-16T09:48:28,064 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T09:48:28,064 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T09:48:28,067 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T09:48:28,067 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731750568067 2024-11-16T09:48:28,067 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731750628067 2024-11-16T09:48:28,067 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T09:48:28,067 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,068 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,068 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,068 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-037c58a19ddb:46839, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,068 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,068 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,071 DEBUG [master/037c58a19ddb:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T09:48:28,074 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.907sec 2024-11-16T09:48:28,074 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T09:48:28,074 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T09:48:28,074 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T09:48:28,074 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T09:48:28,075 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T09:48:28,075 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:28,075 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T09:48:28,076 DEBUG [master/037c58a19ddb:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T09:48:28,076 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T09:48:28,076 INFO [master/037c58a19ddb:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,46839,1731750507025-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:28,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8223428, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:28,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 037c58a19ddb,46839,-1 for getting cluster id 2024-11-16T09:48:28,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T09:48:28,145 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5f2775f2-0950-4ccb-b368-ac3a577fc128' 2024-11-16T09:48:28,146 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T09:48:28,146 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5f2775f2-0950-4ccb-b368-ac3a577fc128" 2024-11-16T09:48:28,146 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623bb4f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:28,146 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [037c58a19ddb,46839,-1] 2024-11-16T09:48:28,147 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T09:48:28,147 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:28,149 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51456, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T09:48:28,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c2ac685, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T09:48:28,151 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:48:28,152 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1] 2024-11-16T09:48:28,153 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:28,155 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:28,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=037c58a19ddb,46839,1731750507025 2024-11-16T09:48:28,158 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T09:48:28,159 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 037c58a19ddb,46839,1731750507025 2024-11-16T09:48:28,159 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58e8be80 2024-11-16T09:48:28,159 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T09:48:28,161 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51470, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T09:48:28,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731750508157', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T09:48:28,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 2024-11-16T09:48:28,165 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T09:48:28,165 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,165 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731750508157" procId is: 4 2024-11-16T09:48:28,166 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T09:48:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:28,178 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8c423c1615b3995b949ed314e576e344, NAME => 'testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731750508157', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:28,186 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:28,187 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1722): Closing 8c423c1615b3995b949ed314e576e344, disabling compactions & flushes 2024-11-16T09:48:28,187 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,187 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,187 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. after waiting 0 ms 2024-11-16T09:48:28,187 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,187 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,187 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731750508157-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8c423c1615b3995b949ed314e576e344: Waiting for close lock at 1731750508186Disabling compacts and flushes for region at 1731750508186Disabling writes for close at 1731750508187 (+1 ms)Writing region close event to WAL at 1731750508187Closed at 1731750508187 2024-11-16T09:48:28,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T09:48:28,188 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731750508188"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731750508188"}]},"ts":"1731750508188"} 2024-11-16T09:48:28,191 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T09:48:28,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T09:48:28,193 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731750508157","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750508192"}]},"ts":"1731750508192"} 2024-11-16T09:48:28,195 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731750508157, state=ENABLING in hbase:meta 2024-11-16T09:48:28,195 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {037c58a19ddb=0} racks are {/default-rack=0} 2024-11-16T09:48:28,196 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T09:48:28,196 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T09:48:28,196 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T09:48:28,196 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T09:48:28,196 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T09:48:28,196 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T09:48:28,196 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T09:48:28,197 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN}] 2024-11-16T09:48:28,199 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN 2024-11-16T09:48:28,200 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN; state=OFFLINE, location=037c58a19ddb,41283,1731750507118; forceNewPlan=false, retain=false 2024-11-16T09:48:28,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:28,351 INFO [037c58a19ddb:46839 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T09:48:28,351 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c423c1615b3995b949ed314e576e344, regionState=OPENING, regionLocation=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:28,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN because future has completed 2024-11-16T09:48:28,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,41283,1731750507118}] 2024-11-16T09:48:28,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:28,510 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,511 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c423c1615b3995b949ed314e576e344, NAME => 'testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:28,511 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731750508157 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,511 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:28,511 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,511 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,512 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,516 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c423c1615b3995b949ed314e576e344 columnFamilyName family 2024-11-16T09:48:28,517 DEBUG [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:28,517 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] regionserver.HStore(327): Store=8c423c1615b3995b949ed314e576e344/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:28,517 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,518 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,518 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,518 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,518 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,519 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,525 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T09:48:28,526 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 8c423c1615b3995b949ed314e576e344; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62276121, jitterRate=-0.0720134824514389}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T09:48:28,526 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:28,527 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 8c423c1615b3995b949ed314e576e344: Running coprocessor pre-open hook at 1731750508512Writing region info on filesystem at 1731750508512Initializing all the Stores at 1731750508512Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750508512Cleaning up temporary data from old regions at 1731750508518 (+6 ms)Running coprocessor post-open hooks at 1731750508526 (+8 ms)Region opened successfully at 1731750508527 (+1 ms) 2024-11-16T09:48:28,527 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344., pid=6, masterSystemTime=1731750508508 2024-11-16T09:48:28,530 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,530 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:28,531 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=8c423c1615b3995b949ed314e576e344, regionState=OPEN, openSeqNum=2, regionLocation=037c58a19ddb,41283,1731750507118 2024-11-16T09:48:28,533 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,41283,1731750507118 because future has completed 2024-11-16T09:48:28,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T09:48:28,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,41283,1731750507118 in 179 msec 2024-11-16T09:48:28,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T09:48:28,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN in 339 msec 2024-11-16T09:48:28,540 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T09:48:28,540 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731750508157","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731750508540"}]},"ts":"1731750508540"} 2024-11-16T09:48:28,542 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731750508157, state=ENABLED in hbase:meta 2024-11-16T09:48:28,543 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T09:48:28,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731750508157 in 380 msec 2024-11-16T09:48:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46839 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T09:48:28,792 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731750508157 completed 2024-11-16T09:48:28,792 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731750508157 get assigned. Timeout = 60000ms 2024-11-16T09:48:28,792 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T09:48:28,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731750508157 assigned to meta. Checking AM states. 2024-11-16T09:48:28,796 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T09:48:28,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731750508157 assigned. 2024-11-16T09:48:28,801 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731750508157', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344., hostname=037c58a19ddb,41283,1731750507118, seqNum=2] 2024-11-16T09:48:28,809 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8c423c1615b3995b949ed314e576e344 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T09:48:28,821 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:28,840 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/.tmp/family/1b872a32179d4875a31e58dbd360bdc8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T09:48:28,841 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/.tmp/family/1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:28,844 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/.tmp/family/1b872a32179d4875a31e58dbd360bdc8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:28,845 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, delay=992 ms 2024-11-16T09:48:28,846 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T09:48:28,847 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 8c423c1615b3995b949ed314e576e344 in 38ms, sequenceid=6, compaction requested=false 2024-11-16T09:48:28,847 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731750508157' 2024-11-16T09:48:28,848 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8c423c1615b3995b949ed314e576e344: 2024-11-16T09:48:29,300 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T09:48:29,838 DEBUG [hfile-prefetch-1731750506452 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1b872a32179d4875a31e58dbd360bdc8_0. 2024-11-16T09:48:29,839 DEBUG [hfile-prefetch-1731750506452 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 1b872a32179d4875a31e58dbd360bdc8_117. 2024-11-16T09:48:29,839 DEBUG [hfile-prefetch-1731750506452 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, with total blocks 2, and data blocks 1 2024-11-16T09:48:29,839 DEBUG [hfile-prefetch-1731750506452 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T09:48:29,839 DEBUG [hfile-prefetch-1731750506452 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T09:48:29,839 DEBUG [hfile-prefetch-1731750506452 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8. Will sleep 100ms and try the verification again. 2024-11-16T09:48:29,849 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;037c58a19ddb:41283,5,FailOnTimeoutGroup] 2024-11-16T09:48:29,849 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,41283,1731750507118' ***** 2024-11-16T09:48:29,849 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-16T09:48:29,849 INFO [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T09:48:29,849 INFO [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence 2024-11-16T09:48:29,850 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T09:48:29,850 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T09:48:29,850 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T09:48:29,850 INFO [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T09:48:29,850 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T09:48:29,860 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1487): Thread RS:1;037c58a19ddb:41283 started persisting bucket cache to file 2024-11-16T09:48:29,860 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:29,860 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:29,863 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-16T09:48:29,863 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731750509860 2024-11-16T09:48:29,863 DEBUG [RS:1;037c58a19ddb:41283 {}] bucket.BucketCache(1509): Thread RS:1;037c58a19ddb:41283 finished persisting bucket cache to file, renaming 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T09:48:29,864 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(3091): Received CLOSE for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,41283,1731750507118 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:29,864 INFO [RS:1;037c58a19ddb:41283 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;037c58a19ddb:41283. 2024-11-16T09:48:29,864 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:29,864 DEBUG [RS:1;037c58a19ddb:41283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:29,865 DEBUG [RS:1;037c58a19ddb:41283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:29,865 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8c423c1615b3995b949ed314e576e344, disabling compactions & flushes 2024-11-16T09:48:29,865 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:29,865 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:29,865 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T09:48:29,865 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. after waiting 0 ms 2024-11-16T09:48:29,865 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T09:48:29,865 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:29,865 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T09:48:29,865 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T09:48:29,865 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T09:48:29,865 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1325): Online Regions={8c423c1615b3995b949ed314e576e344=testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344., 1588230740=hbase:meta,,1.1588230740} 2024-11-16T09:48:29,865 DEBUG [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:29,865 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:29,865 DEBUG [StoreFileCloser-8c423c1615b3995b949ed314e576e344-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:29,865 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:48:29,865 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:48:29,866 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:48:29,866 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T09:48:29,866 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:48:29,865 WARN [hfile-prefetch-1731750506452 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-16T09:48:29,866 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-16T09:48:29,866 DEBUG [hfile-prefetch-1731750506452 {}] hfile.PrefetchExecutor(121): Prefetch completed for 1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:29,871 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T09:48:29,872 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:29,872 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8c423c1615b3995b949ed314e576e344: Waiting for close lock at 1731750509864Running coprocessor pre-close hooks at 1731750509864Disabling compacts and flushes for region at 1731750509864Disabling writes for close at 1731750509865 (+1 ms)Writing region close event to WAL at 1731750509866 (+1 ms)Running coprocessor post-close hooks at 1731750509872 (+6 ms)Closed at 1731750509872 2024-11-16T09:48:29,872 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:29,878 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:29,892 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/d8c5c82d32bd47c2b110d9990a6388dd is 209, key is testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344./info:regioninfo/1731750508530/Put/seqid=0 2024-11-16T09:48:29,893 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:29,897 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T09:48:29,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T09:48:29,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:29,897 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:29,901 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:29,901 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:29,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:29,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750509897 2024-11-16T09:48:29,902 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T09:48:29,915 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/b51491d5ae1e44b08a11a4080e837397 is 43, key is default/ns:d/1731750508059/Put/seqid=0 2024-11-16T09:48:29,916 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:29,925 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on baf1324138f14a289459b5d45e523637 2024-11-16T09:48:29,938 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/baf1324138f14a289459b5d45e523637 is 80, key is testBlockEvictionOnGracefulStop1731750508157/table:state/1731750508540/Put/seqid=0 2024-11-16T09:48:29,938 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/baf1324138f14a289459b5d45e523637 2024-11-16T09:48:29,941 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/d8c5c82d32bd47c2b110d9990a6388dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:29,942 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd, delay=972 ms 2024-11-16T09:48:29,943 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd, entries=10, sequenceid=11, filesize=7.0 K 2024-11-16T09:48:29,944 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/ns/b51491d5ae1e44b08a11a4080e837397 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:29,944 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397, delay=935 ms 2024-11-16T09:48:29,945 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397, entries=2, sequenceid=11, filesize=5.0 K 2024-11-16T09:48:29,946 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/table/baf1324138f14a289459b5d45e523637 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637 2024-11-16T09:48:29,947 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637, delay=925 ms 2024-11-16T09:48:29,948 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637, entries=2, sequenceid=11, filesize=5.3 K 2024-11-16T09:48:29,948 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false 2024-11-16T09:48:29,949 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:29,950 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:29,951 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637 2024-11-16T09:48:29,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-16T09:48:29,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T09:48:29,955 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:29,955 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731750509865Running coprocessor pre-close hooks at 1731750509865Disabling compacts and flushes for region at 1731750509865Disabling writes for close at 1731750509866 (+1 ms)Obtaining lock to block concurrent updates at 1731750509866Preparing flush snapshotting stores in 1588230740 at 1731750509866Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731750509867 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731750509867Flushing 1588230740/info: creating writer at 1731750509867Flushing 1588230740/info: appending metadata at 1731750509892 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731750509892Flushing 1588230740/ns: creating writer at 1731750509895 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731750509915 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731750509915Flushing 1588230740/table: creating writer at 1731750509918 (+3 ms)Flushing 1588230740/table: appending metadata at 1731750509937 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731750509937Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c1f983: reopening flushed file at 1731750509941 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18150acd: reopening flushed file at 1731750509943 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43175f8e: reopening flushed file at 1731750509945 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false at 1731750509948 (+3 ms)Writing region close event to WAL at 1731750509951 (+3 ms)Running coprocessor post-close hooks at 1731750509955 (+4 ms)Closed at 1731750509955 2024-11-16T09:48:29,956 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:30,065 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,41283,1731750507118; all regions closed. 2024-11-16T09:48:30,067 DEBUG [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:30,067 INFO [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C41283%2C1731750507118.meta:.meta(num 1731750508002) 2024-11-16T09:48:30,069 DEBUG [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:30,069 INFO [RS:1;037c58a19ddb:41283 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C41283%2C1731750507118:(num 1731750507649) 2024-11-16T09:48:30,069 DEBUG [RS:1;037c58a19ddb:41283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:30,069 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:30,069 INFO [RS:1;037c58a19ddb:41283 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:30,070 INFO [RS:1;037c58a19ddb:41283 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:30,070 INFO [RS:1;037c58a19ddb:41283 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:30,070 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:30,070 INFO [RS:1;037c58a19ddb:41283 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41283 2024-11-16T09:48:30,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,41283,1731750507118 2024-11-16T09:48:30,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:30,075 INFO [RS:1;037c58a19ddb:41283 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:30,076 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,41283,1731750507118] 2024-11-16T09:48:30,078 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,41283,1731750507118 already deleted, retry=false 2024-11-16T09:48:30,078 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 037c58a19ddb,41283,1731750507118 on 037c58a19ddb,46839,1731750507025 2024-11-16T09:48:30,082 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 037c58a19ddb,41283,1731750507118, splitWal=true, meta=true 2024-11-16T09:48:30,085 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 037c58a19ddb,41283,1731750507118 (carryingMeta=true) 037c58a19ddb,41283,1731750507118/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@20af07fc[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T09:48:30,085 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 037c58a19ddb,41283,1731750507118, splitWal=true, meta=true 2024-11-16T09:48:30,086 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 037c58a19ddb,41283,1731750507118, splitWal=true, meta=true, isMeta: true 2024-11-16T09:48:30,087 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118-splitting 2024-11-16T09:48:30,087 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118-splitting dir is empty, no logs to split. 2024-11-16T09:48:30,088 INFO [PEWorker-2 {}] master.SplitWALManager(105): 037c58a19ddb,41283,1731750507118 WAL count=0, meta=true 2024-11-16T09:48:30,090 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118-splitting dir is empty, no logs to split. 2024-11-16T09:48:30,090 INFO [PEWorker-2 {}] master.SplitWALManager(105): 037c58a19ddb,41283,1731750507118 WAL count=0, meta=true 2024-11-16T09:48:30,090 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 037c58a19ddb,41283,1731750507118 WAL splitting is done? wals=0, meta=true 2024-11-16T09:48:30,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T09:48:30,093 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:30,093 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T09:48:30,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:30,177 INFO [RS:1;037c58a19ddb:41283 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:30,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41283-0x101b07b27660002, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:30,177 INFO [RS:1;037c58a19ddb:41283 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,41283,1731750507118; zookeeper connection closed. 2024-11-16T09:48:30,177 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@64e17b15 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@64e17b15 2024-11-16T09:48:30,244 DEBUG [037c58a19ddb:46839 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-16T09:48:30,245 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=037c58a19ddb,35201,1731750507086 2024-11-16T09:48:30,246 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,35201,1731750507086, state=OPENING 2024-11-16T09:48:30,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:30,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:30,248 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T09:48:30,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:30,248 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:30,248 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,35201,1731750507086}] 2024-11-16T09:48:30,367 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/037c58a19ddb:0 server-side Connection retries=45 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T09:48:30,367 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T09:48:30,368 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40427 2024-11-16T09:48:30,370 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40427 connecting to ZooKeeper ensemble=127.0.0.1:55936 2024-11-16T09:48:30,370 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:30,370 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:30,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:404270x0, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T09:48:30,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40427-0x101b07b27660003 connected 2024-11-16T09:48:30,374 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T09:48:30,374 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T09:48:30,374 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T09:48:30,375 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache 2024-11-16T09:48:30,375 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T09:48:30,375 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T09:48:30,375 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T09:48:30,376 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T09:48:30,376 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T09:48:30,376 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T09:48:30,377 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-16T09:48:30,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T09:48:30,377 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-16T09:48:30,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T09:48:30,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-16T09:48:30,386 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-16T09:48:30,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40427 2024-11-16T09:48:30,387 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T09:48:30,387 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-16T09:48:30,387 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence is complete. 2024-11-16T09:48:30,388 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40427 2024-11-16T09:48:30,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-16T09:48:30,392 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40427 2024-11-16T09:48:30,396 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(746): ClusterId : 5f2775f2-0950-4ccb-b368-ac3a577fc128 2024-11-16T09:48:30,396 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T09:48:30,399 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T09:48:30,399 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T09:48:30,401 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:48:30,401 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T09:48:30,402 DEBUG [RS:2;037c58a19ddb:40427 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6c5688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=037c58a19ddb/172.17.0.2:0 2024-11-16T09:48:30,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41519, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:48:30,410 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T09:48:30,410 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:30,411 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T09:48:30,411 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C35201%2C1731750507086.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,35201,1731750507086, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:30,420 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;037c58a19ddb:40427 2024-11-16T09:48:30,421 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T09:48:30,421 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T09:48:30,421 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T09:48:30,422 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(2659): reportForDuty to master=037c58a19ddb,46839,1731750507025 with port=40427, startcode=1731750510366 2024-11-16T09:48:30,422 DEBUG [RS:2;037c58a19ddb:40427 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T09:48:30,424 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33521, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T09:48:30,425 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,425 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46839 {}] master.ServerManager(517): Registering regionserver=037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,427 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33 2024-11-16T09:48:30,427 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T09:48:30,427 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T09:48:30,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:30,429 DEBUG [RS:2;037c58a19ddb:40427 {}] zookeeper.ZKUtil(111): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,429 WARN [RS:2;037c58a19ddb:40427 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T09:48:30,429 INFO [RS:2;037c58a19ddb:40427 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T09:48:30,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [037c58a19ddb,40427,1731750510366] 2024-11-16T09:48:30,430 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,438 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,35201,1731750507086/037c58a19ddb%2C35201%2C1731750507086.meta.1731750510412.meta 2024-11-16T09:48:30,438 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T09:48:30,440 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:30,440 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:30,440 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T09:48:30,440 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T09:48:30,441 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T09:48:30,441 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T09:48:30,441 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T09:48:30,441 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:30,441 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T09:48:30,441 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T09:48:30,441 INFO [RS:2;037c58a19ddb:40427 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T09:48:30,441 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T09:48:30,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T09:48:30,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:30,443 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T09:48:30,444 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T09:48:30,444 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/037c58a19ddb:0, corePoolSize=2, maxPoolSize=2 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,444 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,445 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,445 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,445 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/037c58a19ddb:0, corePoolSize=1, maxPoolSize=1 2024-11-16T09:48:30,445 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:30,445 DEBUG [RS:2;037c58a19ddb:40427 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/037c58a19ddb:0, corePoolSize=3, maxPoolSize=3 2024-11-16T09:48:30,445 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd, delay=945 ms 2024-11-16T09:48:30,445 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,446 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,446 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,446 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,446 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,446 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,40427,1731750510366-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T09:48:30,447 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:30,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:30,447 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T09:48:30,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T09:48:30,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:30,451 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397, delay=962 ms 2024-11-16T09:48:30,453 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:30,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:30,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T09:48:30,454 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T09:48:30,454 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:30,454 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:30,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T09:48:30,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T09:48:30,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:30,457 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637, delay=913 ms 2024-11-16T09:48:30,458 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637 2024-11-16T09:48:30,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T09:48:30,458 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T09:48:30,460 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:30,460 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740 2024-11-16T09:48:30,461 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T09:48:30,461 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T09:48:30,461 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T09:48:30,463 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T09:48:30,463 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68008337, jitterRate=0.013403192162513733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T09:48:30,463 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T09:48:30,465 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731750510441Writing region info on filesystem at 1731750510441Initializing all the Stores at 1731750510441Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750510441Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750510442 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750510442Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731750510442Cleaning up temporary data from old regions at 1731750510461 (+19 ms)Running coprocessor post-open hooks at 1731750510463 (+2 ms)Region opened successfully at 1731750510465 (+2 ms) 2024-11-16T09:48:30,466 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731750510400 2024-11-16T09:48:30,469 DEBUG [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T09:48:30,469 INFO [RS_OPEN_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T09:48:30,470 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=037c58a19ddb,35201,1731750507086 2024-11-16T09:48:30,470 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 037c58a19ddb,35201,1731750507086, state=OPEN 2024-11-16T09:48:30,472 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T09:48:30,472 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=037c58a19ddb,40427,1731750510366-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:30,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:30,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T09:48:30,472 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=037c58a19ddb,35201,1731750507086 2024-11-16T09:48:30,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:30,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:30,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T09:48:30,472 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,473 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.Replication(171): 037c58a19ddb,40427,1731750510366 started 2024-11-16T09:48:30,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-16T09:48:30,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=037c58a19ddb,35201,1731750507086 in 224 msec 2024-11-16T09:48:30,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T09:48:30,477 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 383 msec 2024-11-16T09:48:30,478 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 037c58a19ddb,41283,1731750507118 had 2 regions 2024-11-16T09:48:30,479 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 037c58a19ddb,41283,1731750507118, splitWal=true, meta=true, isMeta: false 2024-11-16T09:48:30,479 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118-splitting dir is empty, no logs to split. 2024-11-16T09:48:30,479 INFO [PEWorker-3 {}] master.SplitWALManager(105): 037c58a19ddb,41283,1731750507118 WAL count=0, meta=false 2024-11-16T09:48:30,481 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,41283,1731750507118-splitting dir is empty, no logs to split. 2024-11-16T09:48:30,481 INFO [PEWorker-3 {}] master.SplitWALManager(105): 037c58a19ddb,41283,1731750507118 WAL count=0, meta=false 2024-11-16T09:48:30,481 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 037c58a19ddb,41283,1731750507118 WAL splitting is done? wals=0, meta=false 2024-11-16T09:48:30,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN}] 2024-11-16T09:48:30,484 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN 2024-11-16T09:48:30,484 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T09:48:30,494 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T09:48:30,494 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1482): Serving as 037c58a19ddb,40427,1731750510366, RpcServer on 037c58a19ddb/172.17.0.2:40427, sessionid=0x101b07b27660003 2024-11-16T09:48:30,494 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T09:48:30,495 DEBUG [RS:2;037c58a19ddb:40427 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,495 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,40427,1731750510366' 2024-11-16T09:48:30,495 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T09:48:30,495 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '037c58a19ddb,40427,1731750510366' 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T09:48:30,496 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T09:48:30,497 DEBUG [RS:2;037c58a19ddb:40427 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T09:48:30,497 INFO [RS:2;037c58a19ddb:40427 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T09:48:30,497 INFO [RS:2;037c58a19ddb:40427 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T09:48:30,598 INFO [RS:2;037c58a19ddb:40427 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T09:48:30,599 INFO [RS:2;037c58a19ddb:40427 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=037c58a19ddb%2C40427%2C1731750510366, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,40427,1731750510366, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs, maxLogs=55 2024-11-16T09:48:30,626 INFO [RS:2;037c58a19ddb:40427 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/WALs/037c58a19ddb,40427,1731750510366/037c58a19ddb%2C40427%2C1731750510366.1731750510600 2024-11-16T09:48:30,627 DEBUG [RS:2;037c58a19ddb:40427 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(204): Hosts are {037c58a19ddb=0} racks are {/default-rack=0} 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T09:48:30,635 INFO [037c58a19ddb:46839 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T09:48:30,635 INFO [037c58a19ddb:46839 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T09:48:30,635 DEBUG [037c58a19ddb:46839 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T09:48:30,636 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8c423c1615b3995b949ed314e576e344, regionState=OPENING, regionLocation=037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,637 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 037c58a19ddb:41283 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 037c58a19ddb/172.17.0.2:41283 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T09:48:30,639 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1, error=java.net.ConnectException: Call to address=037c58a19ddb:41283 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 037c58a19ddb/172.17.0.2:41283 2024-11-16T09:48:30,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-16T09:48:30,640 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,41283,1731750507118, seqNum=-1 from cache 2024-11-16T09:48:30,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 037c58a19ddb:41283 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 037c58a19ddb/172.17.0.2:41283 2024-11-16T09:48:30,750 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T09:48:30,750 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=037c58a19ddb,35201,1731750507086, seqNum=-1] 2024-11-16T09:48:30,750 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T09:48:30,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47005, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T09:48:30,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN because future has completed 2024-11-16T09:48:30,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,40427,1731750510366}] 2024-11-16T09:48:30,908 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T09:48:30,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60323, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T09:48:30,913 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:30,913 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 8c423c1615b3995b949ed314e576e344, NAME => 'testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.', STARTKEY => '', ENDKEY => ''} 2024-11-16T09:48:30,914 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731750508157 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,914 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T09:48:30,914 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,914 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,915 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,916 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8c423c1615b3995b949ed314e576e344 columnFamilyName family 2024-11-16T09:48:30,916 DEBUG [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T09:48:30,929 DEBUG [StoreFileOpener-8c423c1615b3995b949ed314e576e344-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8, delay=984 ms 2024-11-16T09:48:30,930 DEBUG [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:30,931 INFO [StoreOpener-8c423c1615b3995b949ed314e576e344-1 {}] regionserver.HStore(327): Store=8c423c1615b3995b949ed314e576e344/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T09:48:30,931 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,932 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,932 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,932 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,932 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,934 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,934 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 8c423c1615b3995b949ed314e576e344; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70267176, jitterRate=0.04706251621246338}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T09:48:30,934 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:30,935 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 8c423c1615b3995b949ed314e576e344: Running coprocessor pre-open hook at 1731750510914Writing region info on filesystem at 1731750510914Initializing all the Stores at 1731750510915 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731750510915Cleaning up temporary data from old regions at 1731750510932 (+17 ms)Running coprocessor post-open hooks at 1731750510934 (+2 ms)Region opened successfully at 1731750510935 (+1 ms) 2024-11-16T09:48:30,936 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344., pid=11, masterSystemTime=1731750510908 2024-11-16T09:48:30,939 DEBUG [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:30,939 INFO [RS_OPEN_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:30,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8c423c1615b3995b949ed314e576e344, regionState=OPEN, openSeqNum=10, regionLocation=037c58a19ddb,40427,1731750510366 2024-11-16T09:48:30,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,40427,1731750510366 because future has completed 2024-11-16T09:48:30,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-16T09:48:30,946 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8c423c1615b3995b949ed314e576e344, server=037c58a19ddb,40427,1731750510366 in 188 msec 2024-11-16T09:48:30,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-16T09:48:30,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731750508157, region=8c423c1615b3995b949ed314e576e344, ASSIGN in 464 msec 2024-11-16T09:48:30,948 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 037c58a19ddb,41283,1731750507118 after splitting done 2024-11-16T09:48:30,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 037c58a19ddb,41283,1731750507118, splitWal=true, meta=true in 869 msec 2024-11-16T09:48:30,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T09:48:30,995 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:48:30,995 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:30,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:30,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:30,995 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T09:48:30,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T09:48:30,996 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1622944660, stopped=false 2024-11-16T09:48:30,996 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=037c58a19ddb,46839,1731750507025 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:30,998 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T09:48:30,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:31,002 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T09:48:31,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:31,002 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:31,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:31,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:31,002 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T09:48:31,003 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,35201,1731750507086' ***** 2024-11-16T09:48:31,003 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T09:48:31,003 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '037c58a19ddb,40427,1731750510366' ***** 2024-11-16T09:48:31,003 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T09:48:31,003 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T09:48:31,003 INFO [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T09:48:31,003 INFO [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T09:48:31,003 INFO [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e0710bd6-1cd8-24ab-10c7-cc6f2ac8ca8f/bucket.persistence 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T09:48:31,004 INFO [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T09:48:31,004 INFO [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T09:48:31,004 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T09:48:31,004 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T09:48:31,009 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:31,014 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1487): Thread RS:2;037c58a19ddb:40427 started persisting bucket cache to file 2024-11-16T09:48:31,014 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1487): Thread RS:0;037c58a19ddb:35201 started persisting bucket cache to file 2024-11-16T09:48:31,014 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:31,014 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:31,014 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T09:48:31,014 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T09:48:31,019 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:31,019 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T09:48:31,019 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750511014 2024-11-16T09:48:31,019 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731750511014 2024-11-16T09:48:31,019 DEBUG [RS:0;037c58a19ddb:35201 {}] bucket.BucketCache(1509): Thread RS:0;037c58a19ddb:35201 finished persisting bucket cache to file, renaming 2024-11-16T09:48:31,019 DEBUG [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1509): Thread RS:2;037c58a19ddb:40427 finished persisting bucket cache to file, renaming 2024-11-16T09:48:31,019 WARN [RS:2;037c58a19ddb:40427 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-16T09:48:31,019 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T09:48:31,019 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T09:48:31,019 INFO [RS:0;037c58a19ddb:35201 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T09:48:31,019 INFO [RS:0;037c58a19ddb:35201 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T09:48:31,019 INFO [RS:2;037c58a19ddb:40427 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T09:48:31,019 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,35201,1731750507086 2024-11-16T09:48:31,019 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T09:48:31,019 INFO [RS:0;037c58a19ddb:35201 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:31,019 INFO [RS:2;037c58a19ddb:40427 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;037c58a19ddb:35201. 2024-11-16T09:48:31,020 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(3091): Received CLOSE for 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:31,020 DEBUG [RS:0;037c58a19ddb:35201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:31,020 DEBUG [RS:0;037c58a19ddb:35201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T09:48:31,020 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(959): stopping server 037c58a19ddb,40427,1731750510366 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T09:48:31,020 INFO [RS:2;037c58a19ddb:40427 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:31,020 INFO [RS:2;037c58a19ddb:40427 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;037c58a19ddb:40427. 2024-11-16T09:48:31,020 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:31,020 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T09:48:31,020 DEBUG [RS:2;037c58a19ddb:40427 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T09:48:31,020 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8c423c1615b3995b949ed314e576e344, disabling compactions & flushes 2024-11-16T09:48:31,020 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T09:48:31,020 DEBUG [RS:2;037c58a19ddb:40427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:31,020 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:31,020 DEBUG [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T09:48:31,020 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:31,020 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. after waiting 0 ms 2024-11-16T09:48:31,021 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T09:48:31,021 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:31,021 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1325): Online Regions={8c423c1615b3995b949ed314e576e344=testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344.} 2024-11-16T09:48:31,021 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T09:48:31,021 DEBUG [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1351): Waiting on 8c423c1615b3995b949ed314e576e344 2024-11-16T09:48:31,021 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T09:48:31,021 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T09:48:31,021 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T09:48:31,021 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T09:48:31,021 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T09:48:31,021 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-16T09:48:31,022 DEBUG [StoreFileCloser-8c423c1615b3995b949ed314e576e344-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/family/1b872a32179d4875a31e58dbd360bdc8 2024-11-16T09:48:31,031 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/default/testBlockEvictionOnGracefulStop1731750508157/8c423c1615b3995b949ed314e576e344/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T09:48:31,031 INFO [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:31,031 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8c423c1615b3995b949ed314e576e344: Waiting for close lock at 1731750511020Running coprocessor pre-close hooks at 1731750511020Disabling compacts and flushes for region at 1731750511020Disabling writes for close at 1731750511021 (+1 ms)Writing region close event to WAL at 1731750511022 (+1 ms)Running coprocessor post-close hooks at 1731750511031 (+9 ms)Closed at 1731750511031 2024-11-16T09:48:31,032 DEBUG [RS_CLOSE_REGION-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344. 2024-11-16T09:48:31,036 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5e4d468be14348deaf768fa4755bb5ec 2024-11-16T09:48:31,047 INFO [regionserver/037c58a19ddb:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:31,052 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/5e4d468be14348deaf768fa4755bb5ec is 209, key is testBlockEvictionOnGracefulStop1731750508157,,1731750508162.8c423c1615b3995b949ed314e576e344./info:regioninfo/1731750510939/Put/seqid=0 2024-11-16T09:48:31,053 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/5e4d468be14348deaf768fa4755bb5ec 2024-11-16T09:48:31,056 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/.tmp/info/5e4d468be14348deaf768fa4755bb5ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/5e4d468be14348deaf768fa4755bb5ec 2024-11-16T09:48:31,058 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/5e4d468be14348deaf768fa4755bb5ec, delay=996 ms 2024-11-16T09:48:31,060 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/5e4d468be14348deaf768fa4755bb5ec, entries=8, sequenceid=19, filesize=6.6 K 2024-11-16T09:48:31,061 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 39ms, sequenceid=19, compaction requested=false 2024-11-16T09:48:31,061 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/d8c5c82d32bd47c2b110d9990a6388dd 2024-11-16T09:48:31,062 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/info/5e4d468be14348deaf768fa4755bb5ec 2024-11-16T09:48:31,064 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/ns/b51491d5ae1e44b08a11a4080e837397 2024-11-16T09:48:31,065 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/table/baf1324138f14a289459b5d45e523637 2024-11-16T09:48:31,071 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-16T09:48:31,071 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T09:48:31,072 INFO [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:31,072 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731750511021Running coprocessor pre-close hooks at 1731750511021Disabling compacts and flushes for region at 1731750511021Disabling writes for close at 1731750511021Obtaining lock to block concurrent updates at 1731750511021Preparing flush snapshotting stores in 1588230740 at 1731750511021Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731750511022 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731750511022Flushing 1588230740/info: creating writer at 1731750511022Flushing 1588230740/info: appending metadata at 1731750511051 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731750511051Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@383ee777: reopening flushed file at 1731750511056 (+5 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 39ms, sequenceid=19, compaction requested=false at 1731750511061 (+5 ms)Writing region close event to WAL at 1731750511065 (+4 ms)Running coprocessor post-close hooks at 1731750511071 (+6 ms)Closed at 1731750511071 2024-11-16T09:48:31,072 DEBUG [RS_CLOSE_META-regionserver/037c58a19ddb:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T09:48:31,221 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,35201,1731750507086; all regions closed. 2024-11-16T09:48:31,221 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(976): stopping server 037c58a19ddb,40427,1731750510366; all regions closed. 2024-11-16T09:48:31,223 DEBUG [RS:2;037c58a19ddb:40427 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:31,223 DEBUG [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:31,223 INFO [RS:2;037c58a19ddb:40427 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C40427%2C1731750510366:(num 1731750510600) 2024-11-16T09:48:31,223 DEBUG [RS:2;037c58a19ddb:40427 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:31,223 INFO [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C35201%2C1731750507086.meta:.meta(num 1731750510412) 2024-11-16T09:48:31,223 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:31,223 INFO [RS:2;037c58a19ddb:40427 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:31,223 INFO [RS:2;037c58a19ddb:40427 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:31,224 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T09:48:31,224 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T09:48:31,224 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T09:48:31,224 INFO [RS:2;037c58a19ddb:40427 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:31,224 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:31,224 INFO [RS:2;037c58a19ddb:40427 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40427 2024-11-16T09:48:31,226 DEBUG [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/oldWALs 2024-11-16T09:48:31,226 INFO [RS:0;037c58a19ddb:35201 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 037c58a19ddb%2C35201%2C1731750507086:(num 1731750507654) 2024-11-16T09:48:31,226 DEBUG [RS:0;037c58a19ddb:35201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T09:48:31,226 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T09:48:31,226 INFO [RS:0;037c58a19ddb:35201 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:31,226 INFO [RS:0;037c58a19ddb:35201 {}] hbase.ChoreService(370): Chore service for: regionserver/037c58a19ddb:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:31,227 INFO [RS:0;037c58a19ddb:35201 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:31,227 INFO [RS:0;037c58a19ddb:35201 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35201 2024-11-16T09:48:31,227 INFO [regionserver/037c58a19ddb:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:31,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T09:48:31,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,40427,1731750510366 2024-11-16T09:48:31,237 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,40427,1731750510366] 2024-11-16T09:48:31,237 INFO [RS:2;037c58a19ddb:40427 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:31,237 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/037c58a19ddb,35201,1731750507086 2024-11-16T09:48:31,237 INFO [RS:0;037c58a19ddb:35201 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:31,239 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,40427,1731750510366 already deleted, retry=false 2024-11-16T09:48:31,239 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,40427,1731750510366 expired; onlineServers=1 2024-11-16T09:48:31,239 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [037c58a19ddb,35201,1731750507086] 2024-11-16T09:48:31,241 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/037c58a19ddb,35201,1731750507086 already deleted, retry=false 2024-11-16T09:48:31,241 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 037c58a19ddb,35201,1731750507086 expired; onlineServers=0 2024-11-16T09:48:31,241 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '037c58a19ddb,46839,1731750507025' ***** 2024-11-16T09:48:31,241 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T09:48:31,241 INFO [M:0;037c58a19ddb:46839 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T09:48:31,241 INFO [M:0;037c58a19ddb:46839 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T09:48:31,241 DEBUG [M:0;037c58a19ddb:46839 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T09:48:31,241 DEBUG [M:0;037c58a19ddb:46839 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T09:48:31,241 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T09:48:31,242 INFO [M:0;037c58a19ddb:46839 {}] hbase.ChoreService(370): Chore service for: master/037c58a19ddb:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T09:48:31,242 INFO [M:0;037c58a19ddb:46839 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T09:48:31,242 DEBUG [M:0;037c58a19ddb:46839 {}] master.HMaster(1795): Stopping service threads 2024-11-16T09:48:31,242 INFO [M:0;037c58a19ddb:46839 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T09:48:31,242 INFO [M:0;037c58a19ddb:46839 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T09:48:31,242 INFO [M:0;037c58a19ddb:46839 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T09:48:31,243 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750507371 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.large.0-1731750507371,5,FailOnTimeoutGroup] 2024-11-16T09:48:31,243 DEBUG [master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750507376 {}] cleaner.HFileCleaner(306): Exit Thread[master/037c58a19ddb:0:becomeActiveMaster-HFileCleaner.small.0-1731750507376,5,FailOnTimeoutGroup] 2024-11-16T09:48:31,247 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T09:48:31,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T09:48:31,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T09:48:31,250 DEBUG [M:0;037c58a19ddb:46839 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-16T09:48:31,250 DEBUG [M:0;037c58a19ddb:46839 {}] master.ActiveMasterManager(353): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-16T09:48:31,250 INFO [M:0;037c58a19ddb:46839 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/.lastflushedseqids 2024-11-16T09:48:31,261 INFO [M:0;037c58a19ddb:46839 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T09:48:31,261 INFO [M:0;037c58a19ddb:46839 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T09:48:31,261 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T09:48:31,261 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:31,261 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:31,261 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T09:48:31,261 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:31,262 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-16T09:48:31,274 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ba9de8d7d1744f539450acfb2f469795 2024-11-16T09:48:31,294 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba9de8d7d1744f539450acfb2f469795 is 82, key is hbase:meta,,1/info:regioninfo/1731750510469/Put/seqid=0 2024-11-16T09:48:31,294 INFO [M:0;037c58a19ddb:46839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba9de8d7d1744f539450acfb2f469795 2024-11-16T09:48:31,305 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a137516c0b794bdc95e80d70c43549af 2024-11-16T09:48:31,324 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a137516c0b794bdc95e80d70c43549af is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731750508544/Put/seqid=0 2024-11-16T09:48:31,325 INFO [M:0;037c58a19ddb:46839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a137516c0b794bdc95e80d70c43549af 2024-11-16T09:48:31,335 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 11338e9bb8d5431faaacafbeccfb1c73 2024-11-16T09:48:31,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,340 INFO [RS:2;037c58a19ddb:40427 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:31,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40427-0x101b07b27660003, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,340 INFO [RS:2;037c58a19ddb:40427 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,40427,1731750510366; zookeeper connection closed. 2024-11-16T09:48:31,340 INFO [RS:0;037c58a19ddb:35201 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:31,340 INFO [RS:0;037c58a19ddb:35201 {}] regionserver.HRegionServer(1031): Exiting; stopping=037c58a19ddb,35201,1731750507086; zookeeper connection closed. 2024-11-16T09:48:31,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,340 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35201-0x101b07b27660001, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,340 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7c1490e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7c1490e 2024-11-16T09:48:31,341 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@89468b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@89468b 2024-11-16T09:48:31,341 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T09:48:31,350 DEBUG [M:0;037c58a19ddb:46839 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11338e9bb8d5431faaacafbeccfb1c73 is 69, key is 037c58a19ddb,35201,1731750507086/rs:state/1731750507475/Put/seqid=0 2024-11-16T09:48:31,351 INFO [M:0;037c58a19ddb:46839 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11338e9bb8d5431faaacafbeccfb1c73 2024-11-16T09:48:31,353 INFO [M:0;037c58a19ddb:46839 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11338e9bb8d5431faaacafbeccfb1c73 2024-11-16T09:48:31,354 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ba9de8d7d1744f539450acfb2f469795 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba9de8d7d1744f539450acfb2f469795 2024-11-16T09:48:31,356 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ba9de8d7d1744f539450acfb2f469795, entries=14, sequenceid=102, filesize=5.9 K 2024-11-16T09:48:31,356 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a137516c0b794bdc95e80d70c43549af as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a137516c0b794bdc95e80d70c43549af 2024-11-16T09:48:31,358 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a137516c0b794bdc95e80d70c43549af, entries=11, sequenceid=102, filesize=6.5 K 2024-11-16T09:48:31,359 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/11338e9bb8d5431faaacafbeccfb1c73 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11338e9bb8d5431faaacafbeccfb1c73 2024-11-16T09:48:31,361 INFO [M:0;037c58a19ddb:46839 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11338e9bb8d5431faaacafbeccfb1c73 2024-11-16T09:48:31,361 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/87f69af9-b0a1-08b1-62b3-2ca1c7a85c3d/19ba455d-6ec2-73d0-76ff-3d9833049d33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/11338e9bb8d5431faaacafbeccfb1c73, entries=3, sequenceid=102, filesize=5.3 K 2024-11-16T09:48:31,362 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=102, compaction requested=false 2024-11-16T09:48:31,363 INFO [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T09:48:31,364 DEBUG [M:0;037c58a19ddb:46839 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731750511261Disabling compacts and flushes for region at 1731750511261Disabling writes for close at 1731750511261Obtaining lock to block concurrent updates at 1731750511262 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731750511262Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1731750511262Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731750511262Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731750511262Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731750511293 (+31 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731750511293Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731750511297 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731750511323 (+26 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731750511323Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731750511327 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731750511349 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731750511350 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a0403d: reopening flushed file at 1731750511353 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2af1e549: reopening flushed file at 1731750511356 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5de68be9: reopening flushed file at 1731750511358 (+2 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 100ms, sequenceid=102, compaction requested=false at 1731750511362 (+4 ms)Writing region close event to WAL at 1731750511363 (+1 ms)Closed at 1731750511363 2024-11-16T09:48:31,367 INFO [M:0;037c58a19ddb:46839 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T09:48:31,367 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T09:48:31,367 INFO [M:0;037c58a19ddb:46839 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46839 2024-11-16T09:48:31,368 INFO [M:0;037c58a19ddb:46839 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T09:48:31,470 INFO [M:0;037c58a19ddb:46839 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T09:48:31,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46839-0x101b07b27660000, quorum=127.0.0.1:55936, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T09:48:31,474 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T09:48:31,475 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T09:48:31,476 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T09:48:31,483 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=580 (was 613), ProcessCount=11 (was 11), AvailableMemoryMB=6572 (was 6723)