2024-11-22 23:23:28,907 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-22 23:23:28,925 main DEBUG Took 0.015601 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-22 23:23:28,926 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-22 23:23:28,926 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-22 23:23:28,928 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-22 23:23:28,929 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,951 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-22 23:23:28,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,971 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,972 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,973 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,977 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,979 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,980 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,981 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,982 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,983 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,984 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,985 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,986 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,986 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,987 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,988 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,988 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,989 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,991 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,993 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-22 23:23:28,995 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:28,996 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-22 23:23:28,998 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-22 23:23:29,000 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-22 23:23:29,003 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-22 23:23:29,004 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-22 23:23:29,006 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-22 23:23:29,007 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-22 23:23:29,023 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-22 23:23:29,033 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-22 23:23:29,036 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-22 23:23:29,037 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-22 23:23:29,038 main DEBUG createAppenders(={Console}) 2024-11-22 23:23:29,040 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-22 23:23:29,042 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-22 23:23:29,042 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-22 23:23:29,044 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-22 23:23:29,045 main DEBUG OutputStream closed 2024-11-22 23:23:29,047 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-22 23:23:29,048 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-22 23:23:29,048 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-22 23:23:29,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-22 23:23:29,169 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-22 23:23:29,171 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-22 23:23:29,172 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-22 23:23:29,173 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-22 23:23:29,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-22 23:23:29,174 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-22 23:23:29,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-22 23:23:29,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-22 23:23:29,175 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-22 23:23:29,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-22 23:23:29,176 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-22 23:23:29,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-22 23:23:29,177 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-22 23:23:29,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-22 23:23:29,178 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-22 23:23:29,179 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-22 23:23:29,180 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-22 23:23:29,183 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22 23:23:29,184 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-22 23:23:29,184 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-22 23:23:29,186 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-22T23:23:29,208 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-22 23:23:29,213 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-22 23:23:29,213 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-22T23:23:29,590 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771 2024-11-22T23:23:29,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=384, ProcessCount=11, AvailableMemoryMB=2099 2024-11-22T23:23:29,777 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-22T23:23:29,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:29,914 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/cluster_019ab8af-c0c0-9683-b5b5-8f38e8bbd938, deleteOnExit=true 2024-11-22T23:23:30,148 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/cluster_019ab8af-c0c0-9683-b5b5-8f38e8bbd938/zookeeper_0, clientPort=64774, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/cluster_019ab8af-c0c0-9683-b5b5-8f38e8bbd938/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/cluster_019ab8af-c0c0-9683-b5b5-8f38e8bbd938/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-22T23:23:30,159 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64774 2024-11-22T23:23:30,160 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:30,233 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a with version=8 2024-11-22T23:23:30,234 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/hbase-staging 2024-11-22T23:23:30,318 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-22T23:23:30,561 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3d87a7361f3:0 server-side Connection retries=45 2024-11-22T23:23:30,576 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:30,578 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:30,591 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:23:30,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:30,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:23:30,737 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-22T23:23:30,816 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-22T23:23:30,852 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-22T23:23:30,862 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-22T23:23:30,866 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:23:30,898 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32102 (auto-detected) 2024-11-22T23:23:30,899 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-22T23:23:30,918 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45065 2024-11-22T23:23:30,948 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45065 connecting to ZooKeeper ensemble=127.0.0.1:64774 2024-11-22T23:23:31,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:450650x0, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:23:31,057 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45065-0x10164dc09720000 connected 2024-11-22T23:23:31,155 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:31,158 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:31,170 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:23:31,174 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a, hbase.cluster.distributed=false 2024-11-22T23:23:31,198 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:23:31,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-22T23:23:31,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45065 2024-11-22T23:23:31,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45065 2024-11-22T23:23:31,211 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-22T23:23:31,212 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45065 2024-11-22T23:23:31,301 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3d87a7361f3:0 server-side Connection retries=45 2024-11-22T23:23:31,302 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:31,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:31,303 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-22T23:23:31,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-22T23:23:31,304 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-22T23:23:31,308 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-22T23:23:31,312 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-22T23:23:31,313 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38615 2024-11-22T23:23:31,315 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38615 connecting to ZooKeeper ensemble=127.0.0.1:64774 2024-11-22T23:23:31,316 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:31,317 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:31,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386150x0, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-22T23:23:31,329 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386150x0, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:23:31,329 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38615-0x10164dc09720001 connected 2024-11-22T23:23:31,334 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-22T23:23:31,346 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/bucket.cache 2024-11-22T23:23:31,351 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-22T23:23:31,353 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-22T23:23:31,354 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-22T23:23:31,356 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-22T23:23:31,356 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-22T23:23:31,365 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-22T23:23:31,368 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-22T23:23:31,368 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/bucket.persistence is complete. 2024-11-22T23:23:31,374 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-22T23:23:31,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-22T23:23:31,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-22T23:23:31,384 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38615 2024-11-22T23:23:31,385 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38615 2024-11-22T23:23:31,386 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38615 2024-11-22T23:23:31,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38615 2024-11-22T23:23:31,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38615 2024-11-22T23:23:31,414 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3d87a7361f3:45065 2024-11-22T23:23:31,416 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:23:31,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:23:31,436 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:31,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:31,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-22T23:23:31,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:31,472 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-22T23:23:31,473 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3d87a7361f3,45065,1732317810383 from backup master directory 2024-11-22T23:23:31,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:31,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:23:31,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-22T23:23:31,487 WARN [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:23:31,488 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:31,491 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-22T23:23:31,493 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-22T23:23:31,502 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/hbase.id] with ID: d1c3eaae-4ac8-4d8f-9dba-e57d30124f6d 2024-11-22T23:23:31,503 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/.tmp/hbase.id 2024-11-22T23:23:31,507 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/hbase.id] 2024-11-22T23:23:31,547 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:31,552 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-22T23:23:31,571 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-22T23:23:31,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:31,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:31,613 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:23:31,616 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-22T23:23:31,655 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-22T23:23:31,660 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:23:31,717 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store 2024-11-22T23:23:31,738 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-22T23:23:31,742 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:31,743 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:23:31,743 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:31,744 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:31,745 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:23:31,745 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:31,746 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:31,746 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732317811743Disabling compacts and flushes for region at 1732317811743Disabling writes for close at 1732317811745 (+2 ms)Writing region close event to WAL at 1732317811745Closed at 1732317811745 2024-11-22T23:23:31,747 WARN [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/.initializing 2024-11-22T23:23:31,747 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/WALs/a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:31,758 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:23:31,772 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3d87a7361f3%2C45065%2C1732317810383, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/WALs/a3d87a7361f3,45065,1732317810383, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/oldWALs, maxLogs=10 2024-11-22T23:23:31,809 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/WALs/a3d87a7361f3,45065,1732317810383/a3d87a7361f3%2C45065%2C1732317810383.1732317811776 2024-11-22T23:23:31,809 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:23:31,810 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:23:31,811 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:31,815 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,816 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,851 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,877 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-22T23:23:31,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:31,883 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:31,884 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,888 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-22T23:23:31,888 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:31,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:23:31,889 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,893 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-22T23:23:31,894 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:31,894 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:23:31,895 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-22T23:23:31,899 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:31,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:23:31,901 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,904 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,904 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,907 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,908 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,910 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:23:31,912 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-22T23:23:31,918 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:23:31,920 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65893414, jitterRate=-0.018111616373062134}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:23:31,930 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732317811829Initializing all the Stores at 1732317811829Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317811830 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317811830Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317811831 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317811831Cleaning up temporary data from old regions at 1732317811908 (+77 ms)Region opened successfully at 1732317811930 (+22 ms) 2024-11-22T23:23:31,932 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-22T23:23:31,971 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d8a2ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3d87a7361f3/172.17.0.2:0 2024-11-22T23:23:32,008 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-22T23:23:32,020 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-22T23:23:32,020 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-22T23:23:32,023 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-22T23:23:32,023 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-22T23:23:32,029 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-22T23:23:32,029 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-22T23:23:32,054 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-22T23:23:32,065 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-22T23:23:32,135 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-22T23:23:32,137 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-22T23:23:32,139 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-22T23:23:32,148 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-22T23:23:32,151 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-22T23:23:32,156 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-22T23:23:32,165 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-22T23:23:32,166 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-22T23:23:32,175 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-22T23:23:32,196 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-22T23:23:32,207 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-22T23:23:32,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:23:32,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-22T23:23:32,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,228 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,231 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3d87a7361f3,45065,1732317810383, sessionid=0x10164dc09720000, setting cluster-up flag (Was=false) 2024-11-22T23:23:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,291 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-22T23:23:32,294 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:32,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:32,349 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-22T23:23:32,351 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:32,357 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-22T23:23:32,397 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(746): ClusterId : d1c3eaae-4ac8-4d8f-9dba-e57d30124f6d 2024-11-22T23:23:32,400 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-22T23:23:32,435 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-22T23:23:32,435 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-22T23:23:32,445 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-22T23:23:32,445 DEBUG [RS:0;a3d87a7361f3:38615 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272ae29c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3d87a7361f3/172.17.0.2:0 2024-11-22T23:23:32,449 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-22T23:23:32,459 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3d87a7361f3:38615 2024-11-22T23:23:32,459 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-22T23:23:32,463 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-22T23:23:32,463 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-22T23:23:32,463 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-22T23:23:32,466 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3d87a7361f3,45065,1732317810383 with port=38615, startcode=1732317811265 2024-11-22T23:23:32,466 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-22T23:23:32,471 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3d87a7361f3,45065,1732317810383 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-22T23:23:32,480 DEBUG [RS:0;a3d87a7361f3:38615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-22T23:23:32,482 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:23:32,482 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:23:32,482 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:23:32,482 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=5, maxPoolSize=5 2024-11-22T23:23:32,483 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3d87a7361f3:0, corePoolSize=10, maxPoolSize=10 2024-11-22T23:23:32,483 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,483 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:23:32,483 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,496 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732317842496 2024-11-22T23:23:32,497 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:23:32,498 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-22T23:23:32,499 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-22T23:23:32,501 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-22T23:23:32,504 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-22T23:23:32,504 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:32,504 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-22T23:23:32,505 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-22T23:23:32,505 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-22T23:23:32,504 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-22T23:23:32,514 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,525 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-22T23:23:32,532 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-22T23:23:32,533 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-22T23:23:32,555 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-22T23:23:32,555 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-22T23:23:32,564 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317812556,5,FailOnTimeoutGroup] 2024-11-22T23:23:32,569 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317812564,5,FailOnTimeoutGroup] 2024-11-22T23:23:32,570 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,570 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-22T23:23:32,571 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,572 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,585 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-22T23:23:32,586 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a 2024-11-22T23:23:32,594 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-22T23:23:32,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:32,602 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:23:32,602 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,606 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45065 {}] master.ServerManager(517): Registering regionserver=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,621 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:23:32,621 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a 2024-11-22T23:23:32,621 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-22T23:23:32,621 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:32,621 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-22T23:23:32,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:32,622 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:23:32,639 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:23:32,639 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:32,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:32,640 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:23:32,644 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:23:32,645 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:32,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:32,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:23:32,649 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:23:32,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:32,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:32,650 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:23:32,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740 2024-11-22T23:23:32,652 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740 2024-11-22T23:23:32,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:23:32,653 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:23:32,653 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:23:32,655 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:23:32,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:23:32,662 DEBUG [RS:0;a3d87a7361f3:38615 {}] zookeeper.ZKUtil(111): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,662 WARN [RS:0;a3d87a7361f3:38615 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-22T23:23:32,662 INFO [RS:0;a3d87a7361f3:38615 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:23:32,662 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,669 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3d87a7361f3,38615,1732317811265] 2024-11-22T23:23:32,672 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:23:32,673 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68985357, jitterRate=0.02796192467212677}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:23:32,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732317812600Initializing all the Stores at 1732317812600Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317812600Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317812601 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317812601Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317812601Cleaning up temporary data from old regions at 1732317812653 (+52 ms)Region opened successfully at 1732317812677 (+24 ms) 2024-11-22T23:23:32,677 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:23:32,678 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:23:32,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:23:32,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:23:32,678 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:23:32,693 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:23:32,693 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732317812677Disabling compacts and flushes for region at 1732317812677Disabling writes for close at 1732317812678 (+1 ms)Writing region close event to WAL at 1732317812692 (+14 ms)Closed at 1732317812693 (+1 ms) 2024-11-22T23:23:32,697 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:23:32,698 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-22T23:23:32,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:23:32,709 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-22T23:23:32,717 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:23:32,720 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-22T23:23:32,738 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-22T23:23:32,745 INFO [RS:0;a3d87a7361f3:38615 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-22T23:23:32,745 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,753 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-22T23:23:32,760 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-22T23:23:32,762 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,763 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,763 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,763 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,764 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,764 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,764 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3d87a7361f3:0, corePoolSize=2, maxPoolSize=2 2024-11-22T23:23:32,764 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,764 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,765 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,765 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,765 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,765 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3d87a7361f3:0, corePoolSize=1, maxPoolSize=1 2024-11-22T23:23:32,766 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3d87a7361f3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:23:32,766 DEBUG [RS:0;a3d87a7361f3:38615 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3d87a7361f3:0, corePoolSize=3, maxPoolSize=3 2024-11-22T23:23:32,772 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,773 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,774 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,775 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,775 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,775 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,38615,1732317811265-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:23:32,803 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-22T23:23:32,806 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,38615,1732317811265-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,806 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,807 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.Replication(171): a3d87a7361f3,38615,1732317811265 started 2024-11-22T23:23:32,835 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:32,836 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1482): Serving as a3d87a7361f3,38615,1732317811265, RpcServer on a3d87a7361f3/172.17.0.2:38615, sessionid=0x10164dc09720001 2024-11-22T23:23:32,837 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-22T23:23:32,837 DEBUG [RS:0;a3d87a7361f3:38615 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,838 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3d87a7361f3,38615,1732317811265' 2024-11-22T23:23:32,838 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-22T23:23:32,839 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-22T23:23:32,841 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-22T23:23:32,841 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-22T23:23:32,842 DEBUG [RS:0;a3d87a7361f3:38615 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:32,842 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3d87a7361f3,38615,1732317811265' 2024-11-22T23:23:32,842 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-22T23:23:32,843 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-22T23:23:32,844 DEBUG [RS:0;a3d87a7361f3:38615 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-22T23:23:32,844 INFO [RS:0;a3d87a7361f3:38615 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-22T23:23:32,845 INFO [RS:0;a3d87a7361f3:38615 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-22T23:23:32,872 WARN [a3d87a7361f3:45065 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T23:23:32,952 INFO [RS:0;a3d87a7361f3:38615 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-22T23:23:32,953 INFO [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3d87a7361f3%2C38615%2C1732317811265, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/oldWALs, maxLogs=55 2024-11-22T23:23:32,973 INFO [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265/a3d87a7361f3%2C38615%2C1732317811265.1732317812956 2024-11-22T23:23:32,974 DEBUG [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:23:33,125 DEBUG [a3d87a7361f3:45065 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-22T23:23:33,141 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:33,147 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3d87a7361f3,38615,1732317811265, state=OPENING 2024-11-22T23:23:33,164 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-22T23:23:33,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:33,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:33,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:23:33,176 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:23:33,177 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:23:33,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3d87a7361f3,38615,1732317811265}] 2024-11-22T23:23:33,352 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:23:33,356 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53309, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:23:33,365 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-22T23:23:33,365 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-22T23:23:33,366 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-22T23:23:33,367 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=a3d87a7361f3%2C38615%2C1732317811265.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/oldWALs, maxLogs=55 2024-11-22T23:23:33,387 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265/a3d87a7361f3%2C38615%2C1732317811265.meta.1732317813368.meta 2024-11-22T23:23:33,388 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-22T23:23:33,388 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:23:33,390 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-22T23:23:33,392 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-22T23:23:33,397 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-22T23:23:33,400 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-22T23:23:33,401 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:33,401 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-22T23:23:33,401 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-22T23:23:33,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-22T23:23:33,404 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-22T23:23:33,404 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:33,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:33,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-22T23:23:33,405 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-22T23:23:33,405 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:33,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:33,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-22T23:23:33,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-22T23:23:33,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:33,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:33,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-22T23:23:33,407 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-22T23:23:33,407 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:33,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-22T23:23:33,408 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-22T23:23:33,409 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740 2024-11-22T23:23:33,410 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740 2024-11-22T23:23:33,411 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-22T23:23:33,411 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-22T23:23:33,411 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-22T23:23:33,413 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-22T23:23:33,413 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73807103, jitterRate=0.09981153905391693}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-22T23:23:33,413 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-22T23:23:33,415 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732317813402Writing region info on filesystem at 1732317813402Initializing all the Stores at 1732317813402Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317813402Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317813403 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317813403Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732317813403Cleaning up temporary data from old regions at 1732317813411 (+8 ms)Running coprocessor post-open hooks at 1732317813414 (+3 ms)Region opened successfully at 1732317813415 (+1 ms) 2024-11-22T23:23:33,420 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732317813343 2024-11-22T23:23:33,428 DEBUG [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-22T23:23:33,429 INFO [RS_OPEN_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-22T23:23:33,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:33,431 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3d87a7361f3,38615,1732317811265, state=OPEN 2024-11-22T23:23:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:23:33,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-22T23:23:33,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:23:33,474 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-22T23:23:33,474 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:33,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-22T23:23:33,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3d87a7361f3,38615,1732317811265 in 296 msec 2024-11-22T23:23:33,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-22T23:23:33,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec 2024-11-22T23:23:33,491 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-22T23:23:33,491 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-22T23:23:33,508 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:23:33,509 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,38615,1732317811265, seqNum=-1] 2024-11-22T23:23:33,527 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:23:33,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47765, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:23:33,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1440 sec 2024-11-22T23:23:33,545 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732317813545, completionTime=-1 2024-11-22T23:23:33,548 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-22T23:23:33,549 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-22T23:23:33,572 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-22T23:23:33,573 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732317873573 2024-11-22T23:23:33,573 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732317933573 2024-11-22T23:23:33,573 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-22T23:23:33,575 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,576 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,576 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,577 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3d87a7361f3:45065, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,578 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,579 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,586 DEBUG [master/a3d87a7361f3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-22T23:23:33,605 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.117sec 2024-11-22T23:23:33,607 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-22T23:23:33,609 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-22T23:23:33,610 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-22T23:23:33,611 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-22T23:23:33,611 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-22T23:23:33,612 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-22T23:23:33,613 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-22T23:23:33,618 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-22T23:23:33,618 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-22T23:23:33,619 INFO [master/a3d87a7361f3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3d87a7361f3,45065,1732317810383-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-22T23:23:33,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21abafd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:23:33,713 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-22T23:23:33,713 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-22T23:23:33,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3d87a7361f3,45065,-1 for getting cluster id 2024-11-22T23:23:33,719 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-22T23:23:33,726 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd1c3eaae-4ac8-4d8f-9dba-e57d30124f6d' 2024-11-22T23:23:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-22T23:23:33,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d1c3eaae-4ac8-4d8f-9dba-e57d30124f6d" 2024-11-22T23:23:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1bd0f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:23:33,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3d87a7361f3,45065,-1] 2024-11-22T23:23:33,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-22T23:23:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:33,747 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-22T23:23:33,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e74f0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-22T23:23:33,751 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-22T23:23:33,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3d87a7361f3,38615,1732317811265, seqNum=-1] 2024-11-22T23:23:33,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-22T23:23:33,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-22T23:23:33,783 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:33,786 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-22T23:23:33,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:33,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3f2862ef 2024-11-22T23:23:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-22T23:23:33,797 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-22T23:23:33,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-22T23:23:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-22T23:23:33,809 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-22T23:23:33,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-22T23:23:33,811 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:33,812 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-22T23:23:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:23:33,825 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eda982e42fd814b1d76104e5d2f0efb0, NAME => 'table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a 2024-11-22T23:23:33,831 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:33,831 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing eda982e42fd814b1d76104e5d2f0efb0, disabling compactions & flushes 2024-11-22T23:23:33,832 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:33,832 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:33,832 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. after waiting 0 ms 2024-11-22T23:23:33,832 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:33,832 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:33,832 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for eda982e42fd814b1d76104e5d2f0efb0: Waiting for close lock at 1732317813831Disabling compacts and flushes for region at 1732317813831Disabling writes for close at 1732317813832 (+1 ms)Writing region close event to WAL at 1732317813832Closed at 1732317813832 2024-11-22T23:23:33,833 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-22T23:23:33,838 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1732317813833"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732317813833"}]},"ts":"1732317813833"} 2024-11-22T23:23:33,841 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-22T23:23:33,843 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-22T23:23:33,845 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317813843"}]},"ts":"1732317813843"} 2024-11-22T23:23:33,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-22T23:23:33,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=eda982e42fd814b1d76104e5d2f0efb0, ASSIGN}] 2024-11-22T23:23:33,852 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=eda982e42fd814b1d76104e5d2f0efb0, ASSIGN 2024-11-22T23:23:33,853 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=eda982e42fd814b1d76104e5d2f0efb0, ASSIGN; state=OFFLINE, location=a3d87a7361f3,38615,1732317811265; forceNewPlan=false, retain=false 2024-11-22T23:23:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:23:34,005 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eda982e42fd814b1d76104e5d2f0efb0, regionState=OPENING, regionLocation=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:34,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=eda982e42fd814b1d76104e5d2f0efb0, ASSIGN because future has completed 2024-11-22T23:23:34,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eda982e42fd814b1d76104e5d2f0efb0, server=a3d87a7361f3,38615,1732317811265}] 2024-11-22T23:23:34,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:23:34,166 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:34,167 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eda982e42fd814b1d76104e5d2f0efb0, NAME => 'table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0.', STARTKEY => '', ENDKEY => ''} 2024-11-22T23:23:34,168 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,168 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-22T23:23:34,168 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,168 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,169 INFO [StoreOpener-eda982e42fd814b1d76104e5d2f0efb0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,172 INFO [StoreOpener-eda982e42fd814b1d76104e5d2f0efb0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eda982e42fd814b1d76104e5d2f0efb0 columnFamilyName family 2024-11-22T23:23:34,172 DEBUG [StoreOpener-eda982e42fd814b1d76104e5d2f0efb0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-22T23:23:34,173 INFO [StoreOpener-eda982e42fd814b1d76104e5d2f0efb0-1 {}] regionserver.HStore(327): Store=eda982e42fd814b1d76104e5d2f0efb0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-22T23:23:34,173 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,174 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,175 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,175 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,175 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,176 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,181 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-22T23:23:34,182 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eda982e42fd814b1d76104e5d2f0efb0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68141116, jitterRate=0.01538175344467163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-22T23:23:34,182 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:34,183 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eda982e42fd814b1d76104e5d2f0efb0: Running coprocessor pre-open hook at 1732317814168Writing region info on filesystem at 1732317814168Initializing all the Stores at 1732317814169 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732317814169Cleaning up temporary data from old regions at 1732317814175 (+6 ms)Running coprocessor post-open hooks at 1732317814182 (+7 ms)Region opened successfully at 1732317814183 (+1 ms) 2024-11-22T23:23:34,184 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0., pid=6, masterSystemTime=1732317814163 2024-11-22T23:23:34,187 DEBUG [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:34,187 INFO [RS_OPEN_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:34,187 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eda982e42fd814b1d76104e5d2f0efb0, regionState=OPEN, openSeqNum=2, regionLocation=a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:34,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eda982e42fd814b1d76104e5d2f0efb0, server=a3d87a7361f3,38615,1732317811265 because future has completed 2024-11-22T23:23:34,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-22T23:23:34,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eda982e42fd814b1d76104e5d2f0efb0, server=a3d87a7361f3,38615,1732317811265 in 183 msec 2024-11-22T23:23:34,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-22T23:23:34,199 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=eda982e42fd814b1d76104e5d2f0efb0, ASSIGN in 346 msec 2024-11-22T23:23:34,199 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-22T23:23:34,200 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732317814199"}]},"ts":"1732317814199"} 2024-11-22T23:23:34,203 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-22T23:23:34,204 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-22T23:23:34,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 400 msec 2024-11-22T23:23:34,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45065 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-22T23:23:34,450 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-22T23:23:34,450 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-22T23:23:34,455 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:23:34,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-22T23:23:34,463 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-22T23:23:34,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-22T23:23:34,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0., hostname=a3d87a7361f3,38615,1732317811265, seqNum=2] 2024-11-22T23:23:34,483 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eda982e42fd814b1d76104e5d2f0efb0 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-22T23:23:34,506 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 031ced3c0636457c903dbc3f772cd3cf 2024-11-22T23:23:34,552 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/.tmp/family/031ced3c0636457c903dbc3f772cd3cf is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-22T23:23:34,558 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/.tmp/family/031ced3c0636457c903dbc3f772cd3cf 2024-11-22T23:23:34,598 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/.tmp/family/031ced3c0636457c903dbc3f772cd3cf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf 2024-11-22T23:23:34,600 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf, delay=914 ms 2024-11-22T23:23:34,603 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf, entries=2, sequenceid=6, filesize=4.9 K 2024-11-22T23:23:34,609 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for eda982e42fd814b1d76104e5d2f0efb0 in 124ms, sequenceid=6, compaction requested=false 2024-11-22T23:23:34,611 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-22T23:23:34,613 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eda982e42fd814b1d76104e5d2f0efb0: 2024-11-22T23:23:35,517 DEBUG [hfile-prefetch-1732317814600 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 031ced3c0636457c903dbc3f772cd3cf_0. 2024-11-22T23:23:35,522 DEBUG [hfile-prefetch-1732317814600 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 031ced3c0636457c903dbc3f772cd3cf_117. 2024-11-22T23:23:35,523 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf, with total blocks 2, and data blocks 1 2024-11-22T23:23:35,523 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-22T23:23:35,524 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-22T23:23:35,524 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf. Will sleep 100ms and try the verification again. 2024-11-22T23:23:35,624 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf, with total blocks 2, and data blocks 1 2024-11-22T23:23:35,625 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-22T23:23:35,625 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2293): found block for file 031ced3c0636457c903dbc3f772cd3cf in the backing map. Acquiring read lock for offset 0 2024-11-22T23:23:35,625 DEBUG [hfile-prefetch-1732317814600 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/family/031ced3c0636457c903dbc3f772cd3cf has now been fully cached. 2024-11-22T23:23:35,626 DEBUG [hfile-prefetch-1732317814600 {}] hfile.PrefetchExecutor(121): Prefetch completed for 031ced3c0636457c903dbc3f772cd3cf 2024-11-22T23:23:36,355 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-22T23:23:36,356 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-22T23:23:36,356 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:23:36,356 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-22T23:23:36,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-22T23:23:36,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732317816356 2024-11-22T23:23:36,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-22T23:23:36,619 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-22T23:23:36,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-22T23:23:36,626 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;a3d87a7361f3:38615,5,FailOnTimeoutGroup] 2024-11-22T23:23:36,627 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3d87a7361f3,38615,1732317811265' ***** 2024-11-22T23:23:36,627 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-22T23:23:36,627 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-22T23:23:36,627 INFO [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1738): Disabling cache 2024-11-22T23:23:36,628 INFO [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/bucket.persistence 2024-11-22T23:23:36,628 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-22T23:23:36,628 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-22T23:23:36,628 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-22T23:23:36,628 INFO [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-22T23:23:36,628 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-22T23:23:36,638 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1487): Thread RS:0;a3d87a7361f3:38615 started persisting bucket cache to file 2024-11-22T23:23:36,638 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-22T23:23:36,638 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-22T23:23:36,642 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-22T23:23:36,642 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732317816638 2024-11-22T23:23:36,643 DEBUG [RS:0;a3d87a7361f3:38615 {}] bucket.BucketCache(1509): Thread RS:0;a3d87a7361f3:38615 finished persisting bucket cache to file, renaming 2024-11-22T23:23:36,677 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-22T23:23:36,679 INFO [regionserver/a3d87a7361f3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:23:36,684 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-22T23:23:36,684 INFO [RS:0;a3d87a7361f3:38615 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-22T23:23:36,685 INFO [RS:0;a3d87a7361f3:38615 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-22T23:23:36,685 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(3091): Received CLOSE for eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:36,686 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(959): stopping server a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:36,686 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:23:36,686 INFO [RS:0;a3d87a7361f3:38615 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3d87a7361f3:38615. 2024-11-22T23:23:36,686 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:23:36,686 DEBUG [RS:0;a3d87a7361f3:38615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:23:36,686 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eda982e42fd814b1d76104e5d2f0efb0, disabling compactions & flushes 2024-11-22T23:23:36,687 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:36,687 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:36,687 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. after waiting 0 ms 2024-11-22T23:23:36,687 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:36,694 DEBUG [RS:0;a3d87a7361f3:38615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:36,695 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-22T23:23:36,695 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-22T23:23:36,695 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-22T23:23:36,695 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-22T23:23:36,696 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-22T23:23:36,696 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-22T23:23:36,696 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1325): Online Regions={eda982e42fd814b1d76104e5d2f0efb0=table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0., 1588230740=hbase:meta,,1.1588230740} 2024-11-22T23:23:36,696 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-22T23:23:36,696 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-22T23:23:36,697 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-22T23:23:36,697 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-22T23:23:36,697 DEBUG [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eda982e42fd814b1d76104e5d2f0efb0 2024-11-22T23:23:36,697 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-22T23:23:36,697 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-22T23:23:36,706 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/default/table1/eda982e42fd814b1d76104e5d2f0efb0/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-22T23:23:36,707 INFO [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:36,707 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eda982e42fd814b1d76104e5d2f0efb0: Waiting for close lock at 1732317816686Running coprocessor pre-close hooks at 1732317816686Disabling compacts and flushes for region at 1732317816686Disabling writes for close at 1732317816687 (+1 ms)Writing region close event to WAL at 1732317816690 (+3 ms)Running coprocessor post-close hooks at 1732317816706 (+16 ms)Closed at 1732317816707 (+1 ms) 2024-11-22T23:23:36,708 DEBUG [RS_CLOSE_REGION-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0. 2024-11-22T23:23:36,712 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 17d3e6b1d67b4f50af596aeddb2e9bd0 2024-11-22T23:23:36,728 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/info/17d3e6b1d67b4f50af596aeddb2e9bd0 is 133, key is table1,,1732317813798.eda982e42fd814b1d76104e5d2f0efb0./info:regioninfo/1732317814187/Put/seqid=0 2024-11-22T23:23:36,729 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/info/17d3e6b1d67b4f50af596aeddb2e9bd0 2024-11-22T23:23:36,737 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on cac6fef43a1c45b990b42d7d3def978d 2024-11-22T23:23:36,750 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/ns/cac6fef43a1c45b990b42d7d3def978d is 43, key is default/ns:d/1732317813532/Put/seqid=0 2024-11-22T23:23:36,750 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/ns/cac6fef43a1c45b990b42d7d3def978d 2024-11-22T23:23:36,759 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f7e7017e24e94eefba4d95b6134f0a9d 2024-11-22T23:23:36,772 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/table/f7e7017e24e94eefba4d95b6134f0a9d is 42, key is table1/table:state/1732317814199/Put/seqid=0 2024-11-22T23:23:36,773 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/table/f7e7017e24e94eefba4d95b6134f0a9d 2024-11-22T23:23:36,776 INFO [regionserver/a3d87a7361f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-22T23:23:36,776 INFO [regionserver/a3d87a7361f3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-22T23:23:36,777 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/info/17d3e6b1d67b4f50af596aeddb2e9bd0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/info/17d3e6b1d67b4f50af596aeddb2e9bd0 2024-11-22T23:23:36,778 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/info/17d3e6b1d67b4f50af596aeddb2e9bd0, delay=933 ms 2024-11-22T23:23:36,780 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/info/17d3e6b1d67b4f50af596aeddb2e9bd0, entries=10, sequenceid=11, filesize=6.3 K 2024-11-22T23:23:36,781 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/ns/cac6fef43a1c45b990b42d7d3def978d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/ns/cac6fef43a1c45b990b42d7d3def978d 2024-11-22T23:23:36,783 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/ns/cac6fef43a1c45b990b42d7d3def978d, delay=928 ms 2024-11-22T23:23:36,785 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/ns/cac6fef43a1c45b990b42d7d3def978d, entries=2, sequenceid=11, filesize=5.0 K 2024-11-22T23:23:36,786 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/.tmp/table/f7e7017e24e94eefba4d95b6134f0a9d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/table/f7e7017e24e94eefba4d95b6134f0a9d 2024-11-22T23:23:36,787 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/table/f7e7017e24e94eefba4d95b6134f0a9d, delay=911 ms 2024-11-22T23:23:36,789 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/table/f7e7017e24e94eefba4d95b6134f0a9d, entries=2, sequenceid=11, filesize=5.1 K 2024-11-22T23:23:36,790 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false 2024-11-22T23:23:36,790 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-22T23:23:36,792 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/info/17d3e6b1d67b4f50af596aeddb2e9bd0 2024-11-22T23:23:36,793 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/ns/cac6fef43a1c45b990b42d7d3def978d 2024-11-22T23:23:36,794 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/table/f7e7017e24e94eefba4d95b6134f0a9d 2024-11-22T23:23:36,798 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-22T23:23:36,798 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-22T23:23:36,798 INFO [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-22T23:23:36,798 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732317816696Running coprocessor pre-close hooks at 1732317816696Disabling compacts and flushes for region at 1732317816696Disabling writes for close at 1732317816697 (+1 ms)Obtaining lock to block concurrent updates at 1732317816697Preparing flush snapshotting stores in 1588230740 at 1732317816697Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1732317816698 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732317816698Flushing 1588230740/info: creating writer at 1732317816699 (+1 ms)Flushing 1588230740/info: appending metadata at 1732317816727 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732317816727Flushing 1588230740/ns: creating writer at 1732317816731 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732317816749 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732317816749Flushing 1588230740/table: creating writer at 1732317816753 (+4 ms)Flushing 1588230740/table: appending metadata at 1732317816771 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732317816771Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7da3ffc5: reopening flushed file at 1732317816776 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@161b357f: reopening flushed file at 1732317816781 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2e0ecc: reopening flushed file at 1732317816785 (+4 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=11, compaction requested=false at 1732317816790 (+5 ms)Writing region close event to WAL at 1732317816794 (+4 ms)Running coprocessor post-close hooks at 1732317816798 (+4 ms)Closed at 1732317816798 2024-11-22T23:23:36,799 DEBUG [RS_CLOSE_META-regionserver/a3d87a7361f3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-22T23:23:36,897 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(976): stopping server a3d87a7361f3,38615,1732317811265; all regions closed. 2024-11-22T23:23:36,907 DEBUG [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/oldWALs 2024-11-22T23:23:36,908 INFO [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3d87a7361f3%2C38615%2C1732317811265.meta:.meta(num 1732317813368) 2024-11-22T23:23:36,910 DEBUG [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/oldWALs 2024-11-22T23:23:36,910 INFO [RS:0;a3d87a7361f3:38615 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3d87a7361f3%2C38615%2C1732317811265:(num 1732317812956) 2024-11-22T23:23:36,910 DEBUG [RS:0;a3d87a7361f3:38615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:36,910 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.LeaseManager(133): Closed leases 2024-11-22T23:23:36,910 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:23:36,910 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.ChoreService(370): Chore service for: regionserver/a3d87a7361f3:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-22T23:23:36,911 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:23:36,911 INFO [regionserver/a3d87a7361f3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:23:36,911 INFO [RS:0;a3d87a7361f3:38615 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38615 2024-11-22T23:23:36,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-22T23:23:36,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3d87a7361f3,38615,1732317811265 2024-11-22T23:23:36,927 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:23:36,938 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3d87a7361f3,38615,1732317811265] 2024-11-22T23:23:36,948 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3d87a7361f3,38615,1732317811265 already deleted, retry=false 2024-11-22T23:23:36,948 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of a3d87a7361f3,38615,1732317811265 on a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:36,956 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure a3d87a7361f3,38615,1732317811265, splitWal=true, meta=true 2024-11-22T23:23:36,960 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure a3d87a7361f3,38615,1732317811265, splitWal=true, meta=true 2024-11-22T23:23:36,961 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for a3d87a7361f3,38615,1732317811265 (carryingMeta=true) a3d87a7361f3,38615,1732317811265/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@273aaea0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-22T23:23:36,961 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure a3d87a7361f3,38615,1732317811265, splitWal=true, meta=true, isMeta: true 2024-11-22T23:23:36,962 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265-splitting 2024-11-22T23:23:36,962 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265-splitting dir is empty, no logs to split. 2024-11-22T23:23:36,962 INFO [PEWorker-1 {}] master.SplitWALManager(105): a3d87a7361f3,38615,1732317811265 WAL count=0, meta=true 2024-11-22T23:23:36,964 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/WALs/a3d87a7361f3,38615,1732317811265-splitting dir is empty, no logs to split. 2024-11-22T23:23:36,964 INFO [PEWorker-1 {}] master.SplitWALManager(105): a3d87a7361f3,38615,1732317811265 WAL count=0, meta=true 2024-11-22T23:23:36,964 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if a3d87a7361f3,38615,1732317811265 WAL splitting is done? wals=0, meta=true 2024-11-22T23:23:36,965 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-22T23:23:36,966 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-22T23:23:36,967 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-22T23:23:37,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:23:37,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38615-0x10164dc09720001, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:23:37,039 INFO [RS:0;a3d87a7361f3:38615 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:23:37,039 INFO [RS:0;a3d87a7361f3:38615 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3d87a7361f3,38615,1732317811265; zookeeper connection closed. 2024-11-22T23:23:37,040 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@10e24d65 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@10e24d65 2024-11-22T23:23:37,117 WARN [a3d87a7361f3:45065 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-22T23:23:37,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-22T23:23:37,629 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-22T23:23:37,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:23:37,630 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:37,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:37,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-22T23:23:37,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-22T23:23:37,632 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2117644383, stopped=false 2024-11-22T23:23:37,633 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3d87a7361f3,45065,1732317810383 2024-11-22T23:23:37,633 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'a3d87a7361f3,45065,1732317810383' ***** 2024-11-22T23:23:37,633 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-22T23:23:37,633 INFO [M:0;a3d87a7361f3:45065 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-22T23:23:37,633 INFO [M:0;a3d87a7361f3:45065 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;a3d87a7361f3:45065. 2024-11-22T23:23:37,633 DEBUG [M:0;a3d87a7361f3:45065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-22T23:23:37,633 DEBUG [M:0;a3d87a7361f3:45065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-22T23:23:37,634 INFO [M:0;a3d87a7361f3:45065 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-22T23:23:37,634 DEBUG [M:0;a3d87a7361f3:45065 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-22T23:23:37,634 DEBUG [M:0;a3d87a7361f3:45065 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-22T23:23:37,634 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-22T23:23:37,634 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317812556 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.large.0-1732317812556,5,FailOnTimeoutGroup] 2024-11-22T23:23:37,634 DEBUG [master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317812564 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3d87a7361f3:0:becomeActiveMaster-HFileCleaner.small.0-1732317812564,5,FailOnTimeoutGroup] 2024-11-22T23:23:37,635 INFO [M:0;a3d87a7361f3:45065 {}] hbase.ChoreService(370): Chore service for: master/a3d87a7361f3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-22T23:23:37,635 INFO [M:0;a3d87a7361f3:45065 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-22T23:23:37,635 DEBUG [M:0;a3d87a7361f3:45065 {}] master.HMaster(1795): Stopping service threads 2024-11-22T23:23:37,635 INFO [M:0;a3d87a7361f3:45065 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-22T23:23:37,635 INFO [M:0;a3d87a7361f3:45065 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-22T23:23:37,636 INFO [M:0;a3d87a7361f3:45065 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-22T23:23:37,636 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-22T23:23:37,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-22T23:23:37,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-22T23:23:37,703 DEBUG [M:0;a3d87a7361f3:45065 {}] zookeeper.ZKUtil(347): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-22T23:23:37,703 WARN [M:0;a3d87a7361f3:45065 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-22T23:23:37,704 INFO [M:0;a3d87a7361f3:45065 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/.lastflushedseqids 2024-11-22T23:23:37,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-22T23:23:37,713 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-22T23:23:37,713 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-22T23:23:37,720 INFO [M:0;a3d87a7361f3:45065 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-22T23:23:37,868 DEBUG [a3d87a7361f3:45065 {}] assignment.AssignmentManager(2455): Stopped! Dropping assign of 1 queued regions. 2024-11-22T23:23:37,868 INFO [M:0;a3d87a7361f3:45065 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-22T23:23:37,869 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-22T23:23:37,869 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:37,869 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:37,869 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-22T23:23:37,869 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:37,869 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.18 KB heapSize=30.80 KB 2024-11-22T23:23:37,877 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 308db418555841b3993d631ac1909144 2024-11-22T23:23:37,892 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/308db418555841b3993d631ac1909144 is 82, key is hbase:meta,,1/info:regioninfo/1732317813429/Put/seqid=0 2024-11-22T23:23:37,892 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/308db418555841b3993d631ac1909144 2024-11-22T23:23:37,900 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6bd655d1e5ac49938732edfbb4223bd9 2024-11-22T23:23:37,913 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bd655d1e5ac49938732edfbb4223bd9 is 707, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732317814206/Put/seqid=0 2024-11-22T23:23:37,913 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.57 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bd655d1e5ac49938732edfbb4223bd9 2024-11-22T23:23:37,921 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 86b0f86f5c4b4202a92975b79be2e254 2024-11-22T23:23:37,934 DEBUG [M:0;a3d87a7361f3:45065 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86b0f86f5c4b4202a92975b79be2e254 is 58, key is a3d87a7361f3,38615,1732317811265/rs:/1732317816962/DeleteFamily/seqid=0 2024-11-22T23:23:37,934 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86b0f86f5c4b4202a92975b79be2e254 2024-11-22T23:23:37,936 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 86b0f86f5c4b4202a92975b79be2e254 2024-11-22T23:23:37,937 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/308db418555841b3993d631ac1909144 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/308db418555841b3993d631ac1909144 2024-11-22T23:23:37,938 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/308db418555841b3993d631ac1909144, entries=8, sequenceid=64, filesize=5.5 K 2024-11-22T23:23:37,939 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6bd655d1e5ac49938732edfbb4223bd9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bd655d1e5ac49938732edfbb4223bd9 2024-11-22T23:23:37,940 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6bd655d1e5ac49938732edfbb4223bd9, entries=8, sequenceid=64, filesize=6.5 K 2024-11-22T23:23:37,940 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/86b0f86f5c4b4202a92975b79be2e254 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/86b0f86f5c4b4202a92975b79be2e254 2024-11-22T23:23:37,942 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 86b0f86f5c4b4202a92975b79be2e254 2024-11-22T23:23:37,942 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d59b62c9-0f06-dc4d-74fa-e99586ba7771/3d07d091-02ae-6d9f-e015-d5043db92b5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/86b0f86f5c4b4202a92975b79be2e254, entries=1, sequenceid=64, filesize=5.2 K 2024-11-22T23:23:37,943 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=64, compaction requested=false 2024-11-22T23:23:37,944 INFO [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-22T23:23:37,945 DEBUG [M:0;a3d87a7361f3:45065 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732317817869Disabling compacts and flushes for region at 1732317817869Disabling writes for close at 1732317817869Obtaining lock to block concurrent updates at 1732317817869Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732317817869Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24759, getHeapSize=31480, getOffHeapSize=0, getCellsCount=76 at 1732317817870 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732317817870Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732317817870Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732317817891 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732317817891Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732317817894 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732317817912 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732317817912Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732317817915 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732317817933 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732317817933Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@788b1af9: reopening flushed file at 1732317817936 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@377e94e2: reopening flushed file at 1732317817938 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56fb22a7: reopening flushed file at 1732317817940 (+2 ms)Finished flush of dataSize ~24.18 KB/24759, heapSize ~30.51 KB/31240, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=64, compaction requested=false at 1732317817943 (+3 ms)Writing region close event to WAL at 1732317817944 (+1 ms)Closed at 1732317817944 2024-11-22T23:23:37,946 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-22T23:23:37,946 INFO [M:0;a3d87a7361f3:45065 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-22T23:23:37,947 INFO [M:0;a3d87a7361f3:45065 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45065 2024-11-22T23:23:37,947 INFO [M:0;a3d87a7361f3:45065 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-22T23:23:38,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:23:38,059 INFO [M:0;a3d87a7361f3:45065 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-22T23:23:38,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45065-0x10164dc09720000, quorum=127.0.0.1:64774, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-22T23:23:38,065 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-22T23:23:38,197 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-22T23:23:38,197 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-22T23:23:38,204 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=46 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a3d87a7361f3:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732317816783 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732317814600 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732317816787 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/a3d87a7361f3:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732317816779 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/a3d87a7361f3:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=407 (was 384) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1764 (was 2099)