2024-11-10 04:28:21,544 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-10 04:28:21,556 main DEBUG Took 0.009879 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 04:28:21,556 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 04:28:21,556 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 04:28:21,563 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 04:28:21,564 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,570 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 04:28:21,582 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,583 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,584 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,584 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,584 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,585 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,586 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,586 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,587 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,587 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,588 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,588 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,589 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,589 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,589 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,590 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,590 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,591 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 04:28:21,592 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,593 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 04:28:21,594 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 04:28:21,595 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 04:28:21,597 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 04:28:21,597 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 04:28:21,598 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 04:28:21,598 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 04:28:21,606 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 04:28:21,608 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 04:28:21,610 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 04:28:21,610 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 04:28:21,610 main DEBUG createAppenders(={Console}) 2024-11-10 04:28:21,611 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e initialized 2024-11-10 04:28:21,611 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e 2024-11-10 04:28:21,611 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5745ca0e OK. 2024-11-10 04:28:21,612 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 04:28:21,612 main DEBUG OutputStream closed 2024-11-10 04:28:21,612 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 04:28:21,612 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 04:28:21,613 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@74751b3 OK 2024-11-10 04:28:21,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 04:28:21,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 04:28:21,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 04:28:21,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 04:28:21,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 04:28:21,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 04:28:21,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 04:28:21,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 04:28:21,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 04:28:21,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 04:28:21,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 04:28:21,683 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 04:28:21,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 04:28:21,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 04:28:21,684 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 04:28:21,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 04:28:21,685 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 04:28:21,686 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 04:28:21,688 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 04:28:21,688 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@528c868) with optional ClassLoader: null 2024-11-10 04:28:21,688 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 04:28:21,689 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@528c868] started OK. 2024-11-10T04:28:21,703 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose timeout: 13 mins 2024-11-10 04:28:21,705 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 04:28:21,706 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T04:28:21,923 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db 2024-11-10T04:28:21,950 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=189, ProcessCount=11, AvailableMemoryMB=9490 2024-11-10T04:28:22,046 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T04:28:22,139 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:22,155 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/cluster_cbe7b40f-71c3-cd23-3a9e-ff60fc36f9ef, deleteOnExit=true 2024-11-10T04:28:22,241 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/cluster_cbe7b40f-71c3-cd23-3a9e-ff60fc36f9ef/zookeeper_0, clientPort=49823, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/cluster_cbe7b40f-71c3-cd23-3a9e-ff60fc36f9ef/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/cluster_cbe7b40f-71c3-cd23-3a9e-ff60fc36f9ef/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T04:28:22,253 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49823 2024-11-10T04:28:22,253 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:22,337 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33 with version=8 2024-11-10T04:28:22,337 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/hbase-staging 2024-11-10T04:28:22,434 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T04:28:22,720 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e900e1ae4b9f:0 server-side Connection retries=45 2024-11-10T04:28:22,730 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:22,732 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:22,739 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:28:22,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:22,740 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:28:22,876 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T04:28:22,965 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T04:28:23,005 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T04:28:23,015 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T04:28:23,020 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:28:23,062 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28528 (auto-detected) 2024-11-10T04:28:23,064 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T04:28:23,087 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34951 2024-11-10T04:28:23,119 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34951 connecting to ZooKeeper ensemble=127.0.0.1:49823 2024-11-10T04:28:23,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349510x0, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:28:23,158 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34951-0x101002269dd0000 connected 2024-11-10T04:28:23,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:23,187 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:23,194 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:28:23,198 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33, hbase.cluster.distributed=false 2024-11-10T04:28:23,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:28:23,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34951 2024-11-10T04:28:23,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34951 2024-11-10T04:28:23,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34951 2024-11-10T04:28:23,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34951 2024-11-10T04:28:23,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34951 2024-11-10T04:28:23,329 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e900e1ae4b9f:0 server-side Connection retries=45 2024-11-10T04:28:23,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:23,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:23,331 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T04:28:23,331 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T04:28:23,332 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T04:28:23,335 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T04:28:23,337 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T04:28:23,338 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38243 2024-11-10T04:28:23,340 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38243 connecting to ZooKeeper ensemble=127.0.0.1:49823 2024-11-10T04:28:23,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:23,342 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:23,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382430x0, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T04:28:23,347 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38243-0x101002269dd0001 connected 2024-11-10T04:28:23,347 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:28:23,353 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T04:28:23,366 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/bucket.cache 2024-11-10T04:28:23,372 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T04:28:23,373 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T04:28:23,374 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 1000 2024-11-10T04:28:23,376 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T04:28:23,376 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T04:28:23,381 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T04:28:23,387 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T04:28:23,388 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T04:28:23,388 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/bucket.persistence is complete. 2024-11-10T04:28:23,390 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T04:28:23,395 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T04:28:23,396 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38243 2024-11-10T04:28:23,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38243 2024-11-10T04:28:23,403 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38243 2024-11-10T04:28:23,404 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38243 2024-11-10T04:28:23,405 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38243 2024-11-10T04:28:23,422 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e900e1ae4b9f:34951 2024-11-10T04:28:23,423 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:23,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:28:23,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:28:23,435 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:23,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:23,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T04:28:23,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:23,463 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T04:28:23,465 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e900e1ae4b9f,34951,1731212902513 from backup master directory 2024-11-10T04:28:23,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:28:23,469 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:23,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T04:28:23,470 WARN [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:28:23,471 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:23,473 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T04:28:23,475 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T04:28:23,482 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/hbase.id] with ID: 1882c44b-ccc1-49f3-907f-c911b91f72ad 2024-11-10T04:28:23,482 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/.tmp/hbase.id 2024-11-10T04:28:23,488 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/hbase.id] 2024-11-10T04:28:23,541 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:23,547 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T04:28:23,567 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-10T04:28:23,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:23,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:23,607 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:28:23,613 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T04:28:23,665 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T04:28:23,672 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:28:23,735 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store 2024-11-10T04:28:23,762 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T04:28:23,767 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:23,769 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:28:23,770 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:23,770 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:23,772 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:28:23,772 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:23,773 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:23,773 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212903769Disabling compacts and flushes for region at 1731212903769Disabling writes for close at 1731212903772 (+3 ms)Writing region close event to WAL at 1731212903772Closed at 1731212903773 (+1 ms) 2024-11-10T04:28:23,774 WARN [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/.initializing 2024-11-10T04:28:23,775 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/WALs/e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:23,790 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T04:28:23,806 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e900e1ae4b9f%2C34951%2C1731212902513, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/WALs/e900e1ae4b9f,34951,1731212902513, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/oldWALs, maxLogs=10 2024-11-10T04:28:23,847 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/WALs/e900e1ae4b9f,34951,1731212902513/e900e1ae4b9f%2C34951%2C1731212902513.1731212903811 2024-11-10T04:28:23,848 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T04:28:23,848 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:28:23,849 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:23,853 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,854 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,920 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T04:28:23,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:23,926 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:23,927 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T04:28:23,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:23,934 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:28:23,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T04:28:23,939 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:23,940 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:28:23,940 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,944 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T04:28:23,945 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:23,945 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:28:23,946 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,949 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,950 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,955 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,956 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,960 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:28:23,963 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T04:28:23,973 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:28:23,974 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60642504, jitterRate=-0.09635627269744873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:28:23,984 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731212903866Initializing all the Stores at 1731212903867 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212903867Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212903868 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212903868Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212903868Cleaning up temporary data from old regions at 1731212903956 (+88 ms)Region opened successfully at 1731212903983 (+27 ms) 2024-11-10T04:28:23,986 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T04:28:24,028 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a14e47a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e900e1ae4b9f/172.17.0.2:0 2024-11-10T04:28:24,072 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T04:28:24,084 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T04:28:24,085 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T04:28:24,088 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T04:28:24,088 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T04:28:24,094 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T04:28:24,094 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T04:28:24,120 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T04:28:24,130 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T04:28:24,133 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T04:28:24,135 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T04:28:24,137 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T04:28:24,139 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T04:28:24,142 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T04:28:24,146 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T04:28:24,149 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T04:28:24,151 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T04:28:24,152 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T04:28:24,172 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T04:28:24,174 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T04:28:24,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:28:24,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T04:28:24,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,189 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e900e1ae4b9f,34951,1731212902513, sessionid=0x101002269dd0000, setting cluster-up flag (Was=false) 2024-11-10T04:28:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,214 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T04:28:24,216 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:24,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,228 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T04:28:24,230 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:24,235 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T04:28:24,309 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(746): ClusterId : 1882c44b-ccc1-49f3-907f-c911b91f72ad 2024-11-10T04:28:24,312 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T04:28:24,312 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T04:28:24,317 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T04:28:24,317 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T04:28:24,320 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T04:28:24,321 DEBUG [RS:0;e900e1ae4b9f:38243 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c4a335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e900e1ae4b9f/172.17.0.2:0 2024-11-10T04:28:24,326 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T04:28:24,335 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T04:28:24,336 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e900e1ae4b9f:38243 2024-11-10T04:28:24,339 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T04:28:24,340 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T04:28:24,340 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T04:28:24,343 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(2659): reportForDuty to master=e900e1ae4b9f,34951,1731212902513 with port=38243, startcode=1731212903289 2024-11-10T04:28:24,343 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e900e1ae4b9f,34951,1731212902513 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T04:28:24,351 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e900e1ae4b9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:28:24,351 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e900e1ae4b9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:28:24,352 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e900e1ae4b9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:28:24,352 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e900e1ae4b9f:0, corePoolSize=5, maxPoolSize=5 2024-11-10T04:28:24,352 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e900e1ae4b9f:0, corePoolSize=10, maxPoolSize=10 2024-11-10T04:28:24,352 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,353 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e900e1ae4b9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:28:24,353 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,355 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731212934355 2024-11-10T04:28:24,356 DEBUG [RS:0;e900e1ae4b9f:38243 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T04:28:24,356 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T04:28:24,357 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:28:24,357 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T04:28:24,358 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T04:28:24,361 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T04:28:24,361 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T04:28:24,362 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T04:28:24,362 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T04:28:24,362 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:24,362 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T04:28:24,362 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,369 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T04:28:24,370 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T04:28:24,371 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T04:28:24,373 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T04:28:24,373 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T04:28:24,375 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.large.0-1731212904374,5,FailOnTimeoutGroup] 2024-11-10T04:28:24,375 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.small.0-1731212904375,5,FailOnTimeoutGroup] 2024-11-10T04:28:24,375 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,375 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T04:28:24,377 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,377 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,382 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T04:28:24,383 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33 2024-11-10T04:28:24,392 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:24,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T04:28:24,401 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:28:24,401 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:24,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:24,402 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T04:28:24,406 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:28:24,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:24,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:24,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T04:28:24,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:28:24,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:24,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:24,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T04:28:24,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:28:24,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:24,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:24,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:28:24,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740 2024-11-10T04:28:24,420 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740 2024-11-10T04:28:24,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:28:24,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:28:24,421 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:28:24,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:28:24,428 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58599, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T04:28:24,431 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:28:24,432 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73297166, jitterRate=0.09221288561820984}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:28:24,434 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34951 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731212904392Initializing all the Stores at 1731212904393 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212904393Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212904393Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212904393Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212904393Cleaning up temporary data from old regions at 1731212904421 (+28 ms)Region opened successfully at 1731212904437 (+16 ms) 2024-11-10T04:28:24,437 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34951 {}] master.ServerManager(517): Registering regionserver=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:28:24,437 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:28:24,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:28:24,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:28:24,437 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:28:24,439 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:28:24,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212904437Disabling compacts and flushes for region at 1731212904437Disabling writes for close at 1731212904437Writing region close event to WAL at 1731212904438 (+1 ms)Closed at 1731212904439 (+1 ms) 2024-11-10T04:28:24,442 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:28:24,442 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T04:28:24,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:28:24,451 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33 2024-11-10T04:28:24,451 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T04:28:24,451 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T04:28:24,457 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:28:24,459 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T04:28:24,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:28:24,462 DEBUG [RS:0;e900e1ae4b9f:38243 {}] zookeeper.ZKUtil(111): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,462 WARN [RS:0;e900e1ae4b9f:38243 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T04:28:24,463 INFO [RS:0;e900e1ae4b9f:38243 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:28:24,463 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,464 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e900e1ae4b9f,38243,1731212903289] 2024-11-10T04:28:24,488 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T04:28:24,501 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T04:28:24,506 INFO [RS:0;e900e1ae4b9f:38243 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T04:28:24,506 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,507 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T04:28:24,512 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T04:28:24,513 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,514 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,514 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,514 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,514 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e900e1ae4b9f:0, corePoolSize=2, maxPoolSize=2 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,515 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,516 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e900e1ae4b9f:0, corePoolSize=1, maxPoolSize=1 2024-11-10T04:28:24,516 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e900e1ae4b9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:28:24,516 DEBUG [RS:0;e900e1ae4b9f:38243 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e900e1ae4b9f:0, corePoolSize=3, maxPoolSize=3 2024-11-10T04:28:24,517 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,517 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,517 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,518 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,518 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,518 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,38243,1731212903289-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:28:24,539 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T04:28:24,541 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,38243,1731212903289-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,542 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,542 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.Replication(171): e900e1ae4b9f,38243,1731212903289 started 2024-11-10T04:28:24,561 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:24,562 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1482): Serving as e900e1ae4b9f,38243,1731212903289, RpcServer on e900e1ae4b9f/172.17.0.2:38243, sessionid=0x101002269dd0001 2024-11-10T04:28:24,563 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T04:28:24,563 DEBUG [RS:0;e900e1ae4b9f:38243 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,563 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e900e1ae4b9f,38243,1731212903289' 2024-11-10T04:28:24,563 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T04:28:24,564 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T04:28:24,565 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T04:28:24,565 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T04:28:24,565 DEBUG [RS:0;e900e1ae4b9f:38243 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,566 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e900e1ae4b9f,38243,1731212903289' 2024-11-10T04:28:24,566 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T04:28:24,566 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T04:28:24,567 DEBUG [RS:0;e900e1ae4b9f:38243 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T04:28:24,567 INFO [RS:0;e900e1ae4b9f:38243 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T04:28:24,568 INFO [RS:0;e900e1ae4b9f:38243 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T04:28:24,610 WARN [e900e1ae4b9f:34951 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:28:24,673 INFO [RS:0;e900e1ae4b9f:38243 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T04:28:24,674 INFO [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e900e1ae4b9f%2C38243%2C1731212903289, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/oldWALs, maxLogs=55 2024-11-10T04:28:24,696 INFO [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289/e900e1ae4b9f%2C38243%2C1731212903289.1731212904677 2024-11-10T04:28:24,696 DEBUG [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T04:28:24,862 DEBUG [e900e1ae4b9f:34951 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T04:28:24,875 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:24,881 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e900e1ae4b9f,38243,1731212903289, state=OPENING 2024-11-10T04:28:24,886 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T04:28:24,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:24,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:28:24,889 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:28:24,890 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:28:24,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e900e1ae4b9f,38243,1731212903289}] 2024-11-10T04:28:25,064 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:28:25,068 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37543, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:28:25,076 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T04:28:25,076 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T04:28:25,077 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T04:28:25,078 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=e900e1ae4b9f%2C38243%2C1731212903289.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/oldWALs, maxLogs=55 2024-11-10T04:28:25,099 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289/e900e1ae4b9f%2C38243%2C1731212903289.meta.1731212905080.meta 2024-11-10T04:28:25,100 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T04:28:25,100 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:28:25,102 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T04:28:25,105 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T04:28:25,109 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T04:28:25,113 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T04:28:25,114 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:25,114 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T04:28:25,114 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T04:28:25,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T04:28:25,117 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T04:28:25,117 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:25,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T04:28:25,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T04:28:25,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:25,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T04:28:25,120 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T04:28:25,120 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:25,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T04:28:25,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T04:28:25,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,121 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T04:28:25,122 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T04:28:25,123 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740 2024-11-10T04:28:25,123 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740 2024-11-10T04:28:25,124 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T04:28:25,124 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T04:28:25,124 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T04:28:25,126 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T04:28:25,126 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65035776, jitterRate=-0.03089141845703125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T04:28:25,127 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T04:28:25,128 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731212905115Writing region info on filesystem at 1731212905115Initializing all the Stores at 1731212905115Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212905116 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212905116Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212905116Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731212905116Cleaning up temporary data from old regions at 1731212905124 (+8 ms)Running coprocessor post-open hooks at 1731212905127 (+3 ms)Region opened successfully at 1731212905128 (+1 ms) 2024-11-10T04:28:25,134 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731212905056 2024-11-10T04:28:25,144 DEBUG [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T04:28:25,145 INFO [RS_OPEN_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T04:28:25,146 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:25,147 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e900e1ae4b9f,38243,1731212903289, state=OPEN 2024-11-10T04:28:25,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:28:25,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T04:28:25,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:28:25,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T04:28:25,155 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:25,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T04:28:25,158 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e900e1ae4b9f,38243,1731212903289 in 263 msec 2024-11-10T04:28:25,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T04:28:25,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 710 msec 2024-11-10T04:28:25,163 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T04:28:25,163 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T04:28:25,184 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:28:25,185 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e900e1ae4b9f,38243,1731212903289, seqNum=-1] 2024-11-10T04:28:25,204 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:28:25,208 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55125, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:28:25,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 960 msec 2024-11-10T04:28:25,225 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731212905224, completionTime=-1 2024-11-10T04:28:25,228 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T04:28:25,228 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T04:28:25,256 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T04:28:25,256 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731212965256 2024-11-10T04:28:25,256 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731213025256 2024-11-10T04:28:25,256 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 28 msec 2024-11-10T04:28:25,260 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,260 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,261 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,262 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e900e1ae4b9f:34951, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,263 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,263 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,268 DEBUG [master/e900e1ae4b9f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T04:28:25,288 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.817sec 2024-11-10T04:28:25,290 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T04:28:25,291 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T04:28:25,292 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T04:28:25,293 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T04:28:25,293 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T04:28:25,294 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T04:28:25,295 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T04:28:25,300 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T04:28:25,301 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T04:28:25,301 INFO [master/e900e1ae4b9f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e900e1ae4b9f,34951,1731212902513-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T04:28:25,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567b1492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:28:25,320 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T04:28:25,320 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T04:28:25,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e900e1ae4b9f,34951,-1 for getting cluster id 2024-11-10T04:28:25,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T04:28:25,334 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1882c44b-ccc1-49f3-907f-c911b91f72ad' 2024-11-10T04:28:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T04:28:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1882c44b-ccc1-49f3-907f-c911b91f72ad" 2024-11-10T04:28:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175d2bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:28:25,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e900e1ae4b9f,34951,-1] 2024-11-10T04:28:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T04:28:25,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:25,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T04:28:25,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d5ddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T04:28:25,350 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T04:28:25,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e900e1ae4b9f,38243,1731212903289, seqNum=-1] 2024-11-10T04:28:25,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T04:28:25,360 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46312, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T04:28:25,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:25,384 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T04:28:25,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:25,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2f81d671 2024-11-10T04:28:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T04:28:25,404 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T04:28:25,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'table1', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T04:28:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=table1 2024-11-10T04:28:25,420 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T04:28:25,422 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "table1" procId is: 4 2024-11-10T04:28:25,422 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,424 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T04:28:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:28:25,440 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd2a73f618f991db0a9cdac7b71db797, NAME => 'table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='table1', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33 2024-11-10T04:28:25,447 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(898): Instantiated table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:25,447 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1722): Closing bd2a73f618f991db0a9cdac7b71db797, disabling compactions & flushes 2024-11-10T04:28:25,448 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1755): Closing region table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,448 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,448 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. after waiting 0 ms 2024-11-10T04:28:25,448 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,448 INFO [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1973): Closed table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,448 DEBUG [RegionOpenAndInit-table1-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd2a73f618f991db0a9cdac7b71db797: Waiting for close lock at 1731212905447Disabling compacts and flushes for region at 1731212905447Disabling writes for close at 1731212905448 (+1 ms)Writing region close event to WAL at 1731212905448Closed at 1731212905448 2024-11-10T04:28:25,449 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T04:28:25,454 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.","families":{"info":[{"qualifier":"regioninfo","vlen":40,"tag":[],"timestamp":"1731212905449"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731212905449"}]},"ts":"1731212905449"} 2024-11-10T04:28:25,459 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T04:28:25,460 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T04:28:25,462 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212905460"}]},"ts":"1731212905460"} 2024-11-10T04:28:25,466 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLING in hbase:meta 2024-11-10T04:28:25,468 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=bd2a73f618f991db0a9cdac7b71db797, ASSIGN}] 2024-11-10T04:28:25,469 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=table1, region=bd2a73f618f991db0a9cdac7b71db797, ASSIGN 2024-11-10T04:28:25,471 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=table1, region=bd2a73f618f991db0a9cdac7b71db797, ASSIGN; state=OFFLINE, location=e900e1ae4b9f,38243,1731212903289; forceNewPlan=false, retain=false 2024-11-10T04:28:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:28:25,622 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd2a73f618f991db0a9cdac7b71db797, regionState=OPENING, regionLocation=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:25,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=table1, region=bd2a73f618f991db0a9cdac7b71db797, ASSIGN because future has completed 2024-11-10T04:28:25,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd2a73f618f991db0a9cdac7b71db797, server=e900e1ae4b9f,38243,1731212903289}] 2024-11-10T04:28:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:28:25,783 INFO [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,783 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bd2a73f618f991db0a9cdac7b71db797, NAME => 'table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.', STARTKEY => '', ENDKEY => ''} 2024-11-10T04:28:25,784 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table table1 bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,784 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T04:28:25,784 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,784 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,785 INFO [StoreOpener-bd2a73f618f991db0a9cdac7b71db797-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,789 INFO [StoreOpener-bd2a73f618f991db0a9cdac7b71db797-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd2a73f618f991db0a9cdac7b71db797 columnFamilyName family 2024-11-10T04:28:25,789 DEBUG [StoreOpener-bd2a73f618f991db0a9cdac7b71db797-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T04:28:25,790 INFO [StoreOpener-bd2a73f618f991db0a9cdac7b71db797-1 {}] regionserver.HStore(327): Store=bd2a73f618f991db0a9cdac7b71db797/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T04:28:25,790 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,791 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,791 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,792 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,792 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,792 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,799 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T04:28:25,799 INFO [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bd2a73f618f991db0a9cdac7b71db797; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68962348, jitterRate=0.02761906385421753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T04:28:25,799 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:25,800 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bd2a73f618f991db0a9cdac7b71db797: Running coprocessor pre-open hook at 1731212905784Writing region info on filesystem at 1731212905784Initializing all the Stores at 1731212905785 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731212905785Cleaning up temporary data from old regions at 1731212905792 (+7 ms)Running coprocessor post-open hooks at 1731212905799 (+7 ms)Region opened successfully at 1731212905800 (+1 ms) 2024-11-10T04:28:25,801 INFO [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797., pid=6, masterSystemTime=1731212905780 2024-11-10T04:28:25,805 DEBUG [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,805 INFO [RS_OPEN_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:25,805 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd2a73f618f991db0a9cdac7b71db797, regionState=OPEN, openSeqNum=2, regionLocation=e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:25,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd2a73f618f991db0a9cdac7b71db797, server=e900e1ae4b9f,38243,1731212903289 because future has completed 2024-11-10T04:28:25,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T04:28:25,812 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bd2a73f618f991db0a9cdac7b71db797, server=e900e1ae4b9f,38243,1731212903289 in 183 msec 2024-11-10T04:28:25,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T04:28:25,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=table1, region=bd2a73f618f991db0a9cdac7b71db797, ASSIGN in 343 msec 2024-11-10T04:28:25,815 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T04:28:25,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"table1","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731212905815"}]},"ts":"1731212905815"} 2024-11-10T04:28:25,818 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=table1, state=ENABLED in hbase:meta 2024-11-10T04:28:25,819 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=table1 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T04:28:25,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=table1 in 408 msec 2024-11-10T04:28:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34951 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T04:28:26,056 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table table1 get assigned. Timeout = 60000ms 2024-11-10T04:28:26,056 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:table1 completed 2024-11-10T04:28:26,059 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:28:26,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table table1 assigned to meta. Checking AM states. 2024-11-10T04:28:26,072 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T04:28:26,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table table1 assigned. 2024-11-10T04:28:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'table1', row='row1', locateType=CURRENT is [region=table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797., hostname=e900e1ae4b9f,38243,1731212903289, seqNum=2] 2024-11-10T04:28:26,099 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd2a73f618f991db0a9cdac7b71db797 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T04:28:26,131 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5699eefeead144e69cae30b8846f227f 2024-11-10T04:28:26,183 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/.tmp/family/5699eefeead144e69cae30b8846f227f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T04:28:26,189 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/.tmp/family/5699eefeead144e69cae30b8846f227f 2024-11-10T04:28:26,228 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/.tmp/family/5699eefeead144e69cae30b8846f227f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f 2024-11-10T04:28:26,229 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f, delay=939 ms 2024-11-10T04:28:26,232 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T04:28:26,238 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for bd2a73f618f991db0a9cdac7b71db797 in 140ms, sequenceid=6, compaction requested=false 2024-11-10T04:28:26,239 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'table1' 2024-11-10T04:28:26,241 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd2a73f618f991db0a9cdac7b71db797: 2024-11-10T04:28:27,171 DEBUG [hfile-prefetch-1731212906230 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5699eefeead144e69cae30b8846f227f_0. 2024-11-10T04:28:27,175 DEBUG [hfile-prefetch-1731212906230 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5699eefeead144e69cae30b8846f227f_117. 2024-11-10T04:28:27,176 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f, with total blocks 2, and data blocks 1 2024-11-10T04:28:27,176 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T04:28:27,177 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T04:28:27,177 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f. Will sleep 100ms and try the verification again. 2024-11-10T04:28:27,277 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f, with total blocks 2, and data blocks 1 2024-11-10T04:28:27,278 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-10T04:28:27,278 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2294): found block for file 5699eefeead144e69cae30b8846f227f in the backing map. Acquiring read lock for offset 0 2024-11-10T04:28:27,278 DEBUG [hfile-prefetch-1731212906230 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/family/5699eefeead144e69cae30b8846f227f has now been fully cached. 2024-11-10T04:28:27,279 DEBUG [hfile-prefetch-1731212906230 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5699eefeead144e69cae30b8846f227f 2024-11-10T04:28:27,375 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T04:28:27,375 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T04:28:27,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T04:28:27,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-10T04:28:27,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-10T04:28:27,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731212907375 2024-11-10T04:28:27,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T04:28:28,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T04:28:28,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T04:28:28,250 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;e900e1ae4b9f:38243,5,FailOnTimeoutGroup] 2024-11-10T04:28:28,250 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e900e1ae4b9f,38243,1731212903289' ***** 2024-11-10T04:28:28,250 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-10T04:28:28,250 INFO [Time-limited test {}] hfile.TestPrefetchRSClose(129): Stopped Region Server 0. 2024-11-10T04:28:28,251 INFO [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T04:28:28,251 INFO [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/bucket.persistence 2024-11-10T04:28:28,251 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T04:28:28,251 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T04:28:28,251 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T04:28:28,251 INFO [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T04:28:28,251 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T04:28:28,261 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1488): Thread RS:0;e900e1ae4b9f:38243 started persisting bucket cache to file 2024-11-10T04:28:28,261 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T04:28:28,261 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-10T04:28:28,265 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-10T04:28:28,265 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731212908261 2024-11-10T04:28:28,265 DEBUG [RS:0;e900e1ae4b9f:38243 {}] bucket.BucketCache(1510): Thread RS:0;e900e1ae4b9f:38243 finished persisting bucket cache to file, renaming 2024-11-10T04:28:28,265 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T04:28:28,265 INFO [RS:0;e900e1ae4b9f:38243 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T04:28:28,265 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T04:28:28,266 INFO [RS:0;e900e1ae4b9f:38243 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T04:28:28,266 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(3091): Received CLOSE for bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:28,266 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(959): stopping server e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:28,266 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:28:28,267 INFO [RS:0;e900e1ae4b9f:38243 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e900e1ae4b9f:38243. 2024-11-10T04:28:28,267 INFO [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T04:28:28,267 DEBUG [RS:0;e900e1ae4b9f:38243 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:28:28,267 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd2a73f618f991db0a9cdac7b71db797, disabling compactions & flushes 2024-11-10T04:28:28,267 INFO [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:28,267 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:28,267 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. after waiting 0 ms 2024-11-10T04:28:28,267 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:28,272 DEBUG [RS:0;e900e1ae4b9f:38243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:28,273 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T04:28:28,273 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T04:28:28,273 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T04:28:28,273 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T04:28:28,273 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/default/table1/bd2a73f618f991db0a9cdac7b71db797/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T04:28:28,274 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T04:28:28,274 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T04:28:28,274 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, bd2a73f618f991db0a9cdac7b71db797=table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797.} 2024-11-10T04:28:28,274 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T04:28:28,274 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T04:28:28,274 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T04:28:28,274 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T04:28:28,274 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T04:28:28,274 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.20 KB heapSize=3.23 KB 2024-11-10T04:28:28,275 INFO [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:28,275 DEBUG [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bd2a73f618f991db0a9cdac7b71db797 2024-11-10T04:28:28,275 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd2a73f618f991db0a9cdac7b71db797: Waiting for close lock at 1731212908266Running coprocessor pre-close hooks at 1731212908267 (+1 ms)Disabling compacts and flushes for region at 1731212908267Disabling writes for close at 1731212908267Writing region close event to WAL at 1731212908269 (+2 ms)Running coprocessor post-close hooks at 1731212908274 (+5 ms)Closed at 1731212908275 (+1 ms) 2024-11-10T04:28:28,275 DEBUG [RS_CLOSE_REGION-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797. 2024-11-10T04:28:28,285 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 493ab302038e4428a06c87509eb3d74d 2024-11-10T04:28:28,302 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/info/493ab302038e4428a06c87509eb3d74d is 133, key is table1,,1731212905405.bd2a73f618f991db0a9cdac7b71db797./info:regioninfo/1731212905805/Put/seqid=0 2024-11-10T04:28:28,303 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/info/493ab302038e4428a06c87509eb3d74d 2024-11-10T04:28:28,313 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 33fa95324f0541c0b76bf796eb6f5350 2024-11-10T04:28:28,321 INFO [regionserver/e900e1ae4b9f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:28:28,326 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/ns/33fa95324f0541c0b76bf796eb6f5350 is 43, key is default/ns:d/1731212905212/Put/seqid=0 2024-11-10T04:28:28,327 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/ns/33fa95324f0541c0b76bf796eb6f5350 2024-11-10T04:28:28,336 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b29c1997dbf9488ea57f9bf97eee6593 2024-11-10T04:28:28,350 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/table/b29c1997dbf9488ea57f9bf97eee6593 is 42, key is table1/table:state/1731212905815/Put/seqid=0 2024-11-10T04:28:28,351 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=76 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/table/b29c1997dbf9488ea57f9bf97eee6593 2024-11-10T04:28:28,354 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/info/493ab302038e4428a06c87509eb3d74d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/info/493ab302038e4428a06c87509eb3d74d 2024-11-10T04:28:28,356 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/info/493ab302038e4428a06c87509eb3d74d, delay=963 ms 2024-11-10T04:28:28,358 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/info/493ab302038e4428a06c87509eb3d74d, entries=10, sequenceid=11, filesize=6.3 K 2024-11-10T04:28:28,358 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/ns/33fa95324f0541c0b76bf796eb6f5350 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/ns/33fa95324f0541c0b76bf796eb6f5350 2024-11-10T04:28:28,359 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/ns/33fa95324f0541c0b76bf796eb6f5350, delay=909 ms 2024-11-10T04:28:28,361 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/ns/33fa95324f0541c0b76bf796eb6f5350, entries=2, sequenceid=11, filesize=5.0 K 2024-11-10T04:28:28,361 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/.tmp/table/b29c1997dbf9488ea57f9bf97eee6593 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/table/b29c1997dbf9488ea57f9bf97eee6593 2024-11-10T04:28:28,362 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/table/b29c1997dbf9488ea57f9bf97eee6593, delay=991 ms 2024-11-10T04:28:28,364 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/table/b29c1997dbf9488ea57f9bf97eee6593, entries=2, sequenceid=11, filesize=5.1 K 2024-11-10T04:28:28,365 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false 2024-11-10T04:28:28,365 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T04:28:28,366 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/info/493ab302038e4428a06c87509eb3d74d 2024-11-10T04:28:28,367 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/ns/33fa95324f0541c0b76bf796eb6f5350 2024-11-10T04:28:28,368 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/table/b29c1997dbf9488ea57f9bf97eee6593 2024-11-10T04:28:28,372 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-10T04:28:28,373 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T04:28:28,373 INFO [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T04:28:28,373 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731212908274Running coprocessor pre-close hooks at 1731212908274Disabling compacts and flushes for region at 1731212908274Disabling writes for close at 1731212908274Obtaining lock to block concurrent updates at 1731212908274Preparing flush snapshotting stores in 1588230740 at 1731212908274Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1227, getHeapSize=3248, getOffHeapSize=0, getCellsCount=14 at 1731212908275 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731212908275Flushing 1588230740/info: creating writer at 1731212908276 (+1 ms)Flushing 1588230740/info: appending metadata at 1731212908300 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731212908300Flushing 1588230740/ns: creating writer at 1731212908305 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731212908325 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731212908325Flushing 1588230740/table: creating writer at 1731212908330 (+5 ms)Flushing 1588230740/table: appending metadata at 1731212908349 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731212908349Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@44a15a97: reopening flushed file at 1731212908354 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@47f8823e: reopening flushed file at 1731212908358 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58f3fa48: reopening flushed file at 1731212908361 (+3 ms)Finished flush of dataSize ~1.20 KB/1227, heapSize ~2.94 KB/3008, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=11, compaction requested=false at 1731212908365 (+4 ms)Writing region close event to WAL at 1731212908368 (+3 ms)Running coprocessor post-close hooks at 1731212908372 (+4 ms)Closed at 1731212908373 (+1 ms) 2024-11-10T04:28:28,373 DEBUG [RS_CLOSE_META-regionserver/e900e1ae4b9f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T04:28:28,475 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(976): stopping server e900e1ae4b9f,38243,1731212903289; all regions closed. 2024-11-10T04:28:28,480 DEBUG [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/oldWALs 2024-11-10T04:28:28,480 INFO [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e900e1ae4b9f%2C38243%2C1731212903289.meta:.meta(num 1731212905080) 2024-11-10T04:28:28,482 DEBUG [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/oldWALs 2024-11-10T04:28:28,482 INFO [RS:0;e900e1ae4b9f:38243 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e900e1ae4b9f%2C38243%2C1731212903289:(num 1731212904677) 2024-11-10T04:28:28,482 DEBUG [RS:0;e900e1ae4b9f:38243 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:28,483 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T04:28:28,483 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:28:28,483 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.ChoreService(370): Chore service for: regionserver/e900e1ae4b9f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T04:28:28,483 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:28:28,483 INFO [regionserver/e900e1ae4b9f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:28:28,484 INFO [RS:0;e900e1ae4b9f:38243 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38243 2024-11-10T04:28:28,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e900e1ae4b9f,38243,1731212903289 2024-11-10T04:28:28,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T04:28:28,490 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:28:28,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e900e1ae4b9f,38243,1731212903289] 2024-11-10T04:28:28,494 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e900e1ae4b9f,38243,1731212903289 already deleted, retry=false 2024-11-10T04:28:28,494 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e900e1ae4b9f,38243,1731212903289 on e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:28,499 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e900e1ae4b9f,38243,1731212903289, splitWal=true, meta=true 2024-11-10T04:28:28,501 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e900e1ae4b9f,38243,1731212903289, splitWal=true, meta=true 2024-11-10T04:28:28,502 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=7 for e900e1ae4b9f,38243,1731212903289 (carryingMeta=true) e900e1ae4b9f,38243,1731212903289/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@586e080b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-10T04:28:28,503 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure e900e1ae4b9f,38243,1731212903289, splitWal=true, meta=true, isMeta: true 2024-11-10T04:28:28,503 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289-splitting 2024-11-10T04:28:28,504 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289-splitting dir is empty, no logs to split. 2024-11-10T04:28:28,504 INFO [PEWorker-2 {}] master.SplitWALManager(105): e900e1ae4b9f,38243,1731212903289 WAL count=0, meta=true 2024-11-10T04:28:28,506 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/WALs/e900e1ae4b9f,38243,1731212903289-splitting dir is empty, no logs to split. 2024-11-10T04:28:28,506 INFO [PEWorker-2 {}] master.SplitWALManager(105): e900e1ae4b9f,38243,1731212903289 WAL count=0, meta=true 2024-11-10T04:28:28,506 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if e900e1ae4b9f,38243,1731212903289 WAL splitting is done? wals=0, meta=true 2024-11-10T04:28:28,507 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T04:28:28,508 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T04:28:28,509 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-10T04:28:28,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:28:28,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38243-0x101002269dd0001, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:28:28,594 INFO [RS:0;e900e1ae4b9f:38243 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:28:28,594 INFO [RS:0;e900e1ae4b9f:38243 {}] regionserver.HRegionServer(1031): Exiting; stopping=e900e1ae4b9f,38243,1731212903289; zookeeper connection closed. 2024-11-10T04:28:28,594 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40c600c2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40c600c2 2024-11-10T04:28:28,659 WARN [e900e1ae4b9f:34951 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-10T04:28:29,251 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T04:28:29,251 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T04:28:29,251 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestPrefetchRSClose.tearDown(TestPrefetchRSClose.java:136) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:28:29,251 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:29,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:29,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T04:28:29,252 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T04:28:29,252 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1776925241, stopped=false 2024-11-10T04:28:29,252 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e900e1ae4b9f,34951,1731212902513 2024-11-10T04:28:29,253 INFO [Time-limited test {}] master.HMaster(3321): ***** STOPPING master 'e900e1ae4b9f,34951,1731212902513' ***** 2024-11-10T04:28:29,253 INFO [Time-limited test {}] master.HMaster(3323): STOPPED: OnlineServer=0 right after cluster shutdown set 2024-11-10T04:28:29,253 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T04:28:29,253 INFO [M:0;e900e1ae4b9f:34951 {}] client.AsyncConnectionImpl(233): Connection has been closed by M:0;e900e1ae4b9f:34951. 2024-11-10T04:28:29,253 DEBUG [M:0;e900e1ae4b9f:34951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:630) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T04:28:29,253 DEBUG [M:0;e900e1ae4b9f:34951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T04:28:29,253 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T04:28:29,253 DEBUG [M:0;e900e1ae4b9f:34951 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T04:28:29,253 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T04:28:29,254 DEBUG [M:0;e900e1ae4b9f:34951 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T04:28:29,254 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.small.0-1731212904375 {}] cleaner.HFileCleaner(306): Exit Thread[master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.small.0-1731212904375,5,FailOnTimeoutGroup] 2024-11-10T04:28:29,254 DEBUG [master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.large.0-1731212904374 {}] cleaner.HFileCleaner(306): Exit Thread[master/e900e1ae4b9f:0:becomeActiveMaster-HFileCleaner.large.0-1731212904374,5,FailOnTimeoutGroup] 2024-11-10T04:28:29,254 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.ChoreService(370): Chore service for: master/e900e1ae4b9f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T04:28:29,254 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T04:28:29,254 DEBUG [M:0;e900e1ae4b9f:34951 {}] master.HMaster(1795): Stopping service threads 2024-11-10T04:28:29,254 INFO [M:0;e900e1ae4b9f:34951 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T04:28:29,254 INFO [M:0;e900e1ae4b9f:34951 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T04:28:29,255 INFO [M:0;e900e1ae4b9f:34951 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T04:28:29,255 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T04:28:29,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T04:28:29,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T04:28:29,257 DEBUG [M:0;e900e1ae4b9f:34951 {}] zookeeper.ZKUtil(347): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T04:28:29,257 WARN [M:0;e900e1ae4b9f:34951 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T04:28:29,258 INFO [M:0;e900e1ae4b9f:34951 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/.lastflushedseqids 2024-11-10T04:28:29,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T04:28:29,259 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T04:28:29,260 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T04:28:29,270 INFO [M:0;e900e1ae4b9f:34951 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T04:28:29,410 DEBUG [e900e1ae4b9f:34951 {}] assignment.AssignmentManager(2447): Stopped! Dropping assign of 1 queued regions. 2024-11-10T04:28:29,410 INFO [M:0;e900e1ae4b9f:34951 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T04:28:29,411 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T04:28:29,411 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:29,411 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:29,411 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T04:28:29,411 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:29,411 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=24.16 KB heapSize=30.77 KB 2024-11-10T04:28:29,422 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ed6c2b9f3d1f482b8a2290cca83b82fe 2024-11-10T04:28:29,435 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed6c2b9f3d1f482b8a2290cca83b82fe is 82, key is hbase:meta,,1/info:regioninfo/1731212905145/Put/seqid=0 2024-11-10T04:28:29,436 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed6c2b9f3d1f482b8a2290cca83b82fe 2024-11-10T04:28:29,444 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74a9387a0de3493392cab6e43751161a 2024-11-10T04:28:29,458 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74a9387a0de3493392cab6e43751161a is 705, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731212905820/Put/seqid=0 2024-11-10T04:28:29,458 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=23.55 KB at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74a9387a0de3493392cab6e43751161a 2024-11-10T04:28:29,466 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d31a5651d99e4965953d2884d1a23f60 2024-11-10T04:28:29,480 DEBUG [M:0;e900e1ae4b9f:34951 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d31a5651d99e4965953d2884d1a23f60 is 58, key is e900e1ae4b9f,38243,1731212903289/rs:/1731212908503/DeleteFamily/seqid=0 2024-11-10T04:28:29,480 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=119 B at sequenceid=64 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d31a5651d99e4965953d2884d1a23f60 2024-11-10T04:28:29,482 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d31a5651d99e4965953d2884d1a23f60 2024-11-10T04:28:29,483 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ed6c2b9f3d1f482b8a2290cca83b82fe as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ed6c2b9f3d1f482b8a2290cca83b82fe 2024-11-10T04:28:29,484 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ed6c2b9f3d1f482b8a2290cca83b82fe, entries=8, sequenceid=64, filesize=5.5 K 2024-11-10T04:28:29,485 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/74a9387a0de3493392cab6e43751161a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74a9387a0de3493392cab6e43751161a 2024-11-10T04:28:29,486 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/74a9387a0de3493392cab6e43751161a, entries=8, sequenceid=64, filesize=6.5 K 2024-11-10T04:28:29,487 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d31a5651d99e4965953d2884d1a23f60 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d31a5651d99e4965953d2884d1a23f60 2024-11-10T04:28:29,489 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d31a5651d99e4965953d2884d1a23f60 2024-11-10T04:28:29,489 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0b4b6749-1468-4033-4f0c-38f9615913db/4dd09d82-bf89-fe3a-757e-aafbaf2adc33/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d31a5651d99e4965953d2884d1a23f60, entries=1, sequenceid=64, filesize=5.2 K 2024-11-10T04:28:29,490 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(3140): Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=64, compaction requested=false 2024-11-10T04:28:29,492 INFO [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T04:28:29,492 DEBUG [M:0;e900e1ae4b9f:34951 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731212909411Disabling compacts and flushes for region at 1731212909411Disabling writes for close at 1731212909411Obtaining lock to block concurrent updates at 1731212909411Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731212909411Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=24735, getHeapSize=31448, getOffHeapSize=0, getCellsCount=76 at 1731212909412 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731212909412Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731212909412Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731212909435 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731212909435Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731212909438 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731212909457 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731212909457Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731212909460 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731212909479 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731212909479Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bdf8d92: reopening flushed file at 1731212909483 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31aab6fa: reopening flushed file at 1731212909485 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@608d0f04: reopening flushed file at 1731212909487 (+2 ms)Finished flush of dataSize ~24.16 KB/24735, heapSize ~30.48 KB/31208, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=64, compaction requested=false at 1731212909490 (+3 ms)Writing region close event to WAL at 1731212909491 (+1 ms)Closed at 1731212909491 2024-11-10T04:28:29,493 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T04:28:29,493 INFO [M:0;e900e1ae4b9f:34951 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T04:28:29,493 INFO [M:0;e900e1ae4b9f:34951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34951 2024-11-10T04:28:29,494 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T04:28:29,596 INFO [M:0;e900e1ae4b9f:34951 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T04:28:29,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:28:29,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34951-0x101002269dd0000, quorum=127.0.0.1:49823, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T04:28:29,600 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T04:28:29,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T04:28:29,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T04:28:29,632 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestPrefetchRSClose#testPrefetchPersistence Thread=44 (was 12) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731212906230 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731212908363 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731212908356 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/e900e1ae4b9f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731212908360 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e900e1ae4b9f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/e900e1ae4b9f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) - Thread LEAK? -, OpenFileDescriptor=337 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=173 (was 189), ProcessCount=11 (was 11), AvailableMemoryMB=9170 (was 9490)